diff --git a/.github/workflows/code-checker.yaml b/.github/workflows/code-checker.yaml index d4f4416745..21c5df8c94 100644 --- a/.github/workflows/code-checker.yaml +++ b/.github/workflows/code-checker.yaml @@ -35,6 +35,14 @@ jobs: strategy: fail-fast: false steps: + - name: Maximize build space + uses: easimon/maximize-build-space@master + with: + root-reserve-mb: 10240 + swap-size-mb: 1024 + remove-dotnet: 'true' + remove-android: 'true' + remove-haskell: 'true' - name: Checkout uses: actions/checkout@v2 - name: 'Generate CCache Hash' @@ -74,6 +82,14 @@ jobs: runs-on: ubuntu-latest timeout-minutes: 180 steps: + - name: Maximize build space + uses: easimon/maximize-build-space@master + with: + root-reserve-mb: 10240 + swap-size-mb: 1024 + remove-dotnet: 'true' + remove-android: 'true' + remove-haskell: 'true' - name: Checkout uses: actions/checkout@v2 - name: 'Generate CCache Hash' diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 590e0b1cf1..fbec3c4235 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -47,6 +47,14 @@ jobs: env: UBUNTU: ${{ matrix.ubuntu }} steps: + - name: Maximize build space + uses: easimon/maximize-build-space@master + with: + root-reserve-mb: 10240 + swap-size-mb: 1024 + remove-dotnet: 'true' + remove-android: 'true' + remove-haskell: 'true' - name: Checkout uses: actions/checkout@v2 with: @@ -92,7 +100,7 @@ jobs: run: | ./build/builder.sh /bin/bash -c "make USE_ASAN=${{env.useasan}} build-cpp-with-coverage" - run: | - zip -r code.zip . -x "./.docker/*" + zip -r code.zip . -x "./.docker/*" -x "./cmake_build/thirdparty/*" - name: Archive code uses: actions/upload-artifact@v3 with: @@ -127,8 +135,6 @@ jobs: shell: bash run: | docker-compose up -d azurite -# - name: 'Setup upterm session' -# uses: lhotari/action-upterm@v1 - name: UT run: | chmod +x build/builder.sh @@ -173,8 +179,6 @@ jobs: shell: bash run: | docker-compose up -d pulsar etcd minio azurite -# - name: 'Setup upterm session' -# uses: lhotari/action-upterm@v1 - name: UT run: | chmod +x build/builder.sh diff --git a/configs/milvus.yaml b/configs/milvus.yaml index 838b4c00a2..83ab2a4c0e 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -91,7 +91,7 @@ minio: # Supported level: off, fatal, error, warn, info, debug, trace logLevel: fatal # Cloud data center region - region: "" + region: '' # Cloud whether use virtual host bucket mode useVirtualHost: false # timeout for request time in milliseconds @@ -500,7 +500,7 @@ common: BeamWidthRatio: 4 gracefulTime: 5000 # milliseconds. it represents the interval (in ms) by which the request arrival time needs to be subtracted in the case of Bounded Consistency. gracefulStopTimeout: 1800 # seconds. it will force quit the server if the graceful stop process is not completed during this time. - storageType: minio # please adjust in embedded Milvus: local + storageType: opendal # please adjust in embedded Milvus: local # Default value: auto # Valid values: [auto, avx512, avx2, avx, sse4_2] # This configuration is only used by querynode and indexnode, it selects CPU instruction set for Searching and Index-building. @@ -520,9 +520,9 @@ common: enabled: false # support pre-created topics # the name of pre-created topics - names: ["topic1", "topic2"] + names: ['topic1', 'topic2'] # need to set a separated topic to stand for currently consumed timestamp for each channel - timeticker: "timetick-channel" + timeticker: 'timetick-channel' ImportMaxFileSize: 17179869184 # 16 * 1024 * 1024 * 1024 # max file size to import for bulkInsert diff --git a/internal/core/conanfile.py b/internal/core/conanfile.py index 69c6e535f3..650e9135d2 100644 --- a/internal/core/conanfile.py +++ b/internal/core/conanfile.py @@ -6,7 +6,7 @@ class MilvusConan(ConanFile): requires = ( "rocksdb/6.29.5", "boost/1.82.0", - "onetbb/2021.7.0", + "onetbb/2021.9.0", "nlohmann_json/3.11.2", "zstd/1.5.4", "lz4/1.9.4", diff --git a/internal/core/src/common/CMakeLists.txt b/internal/core/src/common/CMakeLists.txt index 7fdac692ed..5072728c20 100644 --- a/internal/core/src/common/CMakeLists.txt +++ b/internal/core/src/common/CMakeLists.txt @@ -33,6 +33,7 @@ target_link_libraries(milvus_common yaml-cpp boost_bitset_ext simdjson + opendal ${CONAN_LIBS} ) diff --git a/internal/core/src/indexbuilder/CMakeLists.txt b/internal/core/src/indexbuilder/CMakeLists.txt index cae7415c9c..a6fdea6456 100644 --- a/internal/core/src/indexbuilder/CMakeLists.txt +++ b/internal/core/src/indexbuilder/CMakeLists.txt @@ -20,17 +20,7 @@ set(INDEXBUILDER_FILES milvus_add_pkg_config("milvus_indexbuilder") add_library(milvus_indexbuilder SHARED ${INDEXBUILDER_FILES}) -find_library(TBB NAMES tbb) -set(PLATFORM_LIBS dl) -if (MSYS) -set(PLATFORM_LIBS -Wl,--allow-multiple-definition) -endif () - # link order matters -target_link_libraries(milvus_indexbuilder - milvus_index - ${TBB} - ${PLATFORM_LIBS} - ) +target_link_libraries(milvus_indexbuilder milvus_index) install(TARGETS milvus_indexbuilder DESTINATION "${CMAKE_INSTALL_LIBDIR}") diff --git a/internal/core/src/indexbuilder/index_c.cpp b/internal/core/src/indexbuilder/index_c.cpp index 0052ea329b..62aad7d267 100644 --- a/internal/core/src/indexbuilder/index_c.cpp +++ b/internal/core/src/indexbuilder/index_c.cpp @@ -310,6 +310,8 @@ NewBuildIndexInfo(CBuildIndexInfo* c_build_index_info, std::string(c_storage_config.cloud_provider); storage_config.iam_endpoint = std::string(c_storage_config.iam_endpoint); + storage_config.cloud_provider = + std::string(c_storage_config.cloud_provider); storage_config.useSSL = c_storage_config.useSSL; storage_config.useIAM = c_storage_config.useIAM; storage_config.region = c_storage_config.region; diff --git a/internal/core/src/segcore/CMakeLists.txt b/internal/core/src/segcore/CMakeLists.txt index af04c11d6c..449ae24729 100644 --- a/internal/core/src/segcore/CMakeLists.txt +++ b/internal/core/src/segcore/CMakeLists.txt @@ -41,18 +41,6 @@ set(SEGCORE_FILES ConcurrentVector.cpp) add_library(milvus_segcore SHARED ${SEGCORE_FILES}) -find_library(TBB NAMES tbb) -set(PLATFORM_LIBS dl) - -if (MSYS) - set(PLATFORM_LIBS ) -endif() - -target_link_libraries(milvus_segcore - milvus_query - ${PLATFORM_LIBS} - ${TBB} - ${OpenMP_CXX_FLAGS} - ) +target_link_libraries(milvus_segcore milvus_query ${OpenMP_CXX_FLAGS}) install(TARGETS milvus_segcore DESTINATION "${CMAKE_INSTALL_LIBDIR}") diff --git a/internal/core/src/storage/CMakeLists.txt b/internal/core/src/storage/CMakeLists.txt index 2e698161b9..cc3539c75c 100644 --- a/internal/core/src/storage/CMakeLists.txt +++ b/internal/core/src/storage/CMakeLists.txt @@ -47,8 +47,9 @@ set(STORAGE_FILES Event.cpp ThreadPool.cpp storage_c.cpp + ChunkManager.cpp MinioChunkManager.cpp - ChunkManagers.cpp + OpenDALChunkManager.cpp AliyunSTSClient.cpp AliyunCredentialsProvider.cpp MemFileManagerImpl.cpp diff --git a/internal/core/src/storage/ChunkManagers.cpp b/internal/core/src/storage/ChunkManager.cpp similarity index 100% rename from internal/core/src/storage/ChunkManagers.cpp rename to internal/core/src/storage/ChunkManager.cpp diff --git a/internal/core/src/storage/ChunkManager.h b/internal/core/src/storage/ChunkManager.h index bec5addd2a..6b0cfb8091 100644 --- a/internal/core/src/storage/ChunkManager.h +++ b/internal/core/src/storage/ChunkManager.h @@ -129,6 +129,7 @@ enum class ChunkManagerType : int8_t { Local = 1, Minio = 2, Remote = 3, + OpenDAL = 4, }; extern std::map ChunkManagerType_Map; diff --git a/internal/core/src/storage/OpenDALChunkManager.cpp b/internal/core/src/storage/OpenDALChunkManager.cpp new file mode 100644 index 0000000000..9945de7984 --- /dev/null +++ b/internal/core/src/storage/OpenDALChunkManager.cpp @@ -0,0 +1,206 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include +#include +#include + +#include "log/Log.h" +#include "opendal.h" +#include "common/EasyAssert.h" +#include "storage/Util.h" +#include "storage/OpenDALChunkManager.h" + +namespace milvus::storage { + +std::string +ToString(opendal_bytes* bs) { + return {reinterpret_cast(bs->data), bs->len}; +} + +#define THROWOPENDALERROR(err, msg) \ + do { \ + auto exception = SegcoreError( \ + S3Error, fmt::format("{}: {}", (msg), ToString(&(err)->message))); \ + opendal_error_free(err); \ + throw exception; \ + } while (0) + +// std::once_flag init_flag_; + +OpenDALChunkManager::OpenDALChunkManager(const StorageConfig& storage_config) + : default_bucket_name_(storage_config.bucket_name) { + // std::call_once(init_flag_, []() { opendal_init_logger(); }); + remote_root_path_ = storage_config.root_path; + std::string storageType; + if (storage_config.cloud_provider == "gcp") { + storageType = "gcs"; + } else if (storage_config.cloud_provider == "aliyun") { + storageType = "oss"; + } else if (storage_config.cloud_provider == "azure") { + storageType = "azblob"; + } else { + storageType = "s3"; + } + + opendal_operator_options* op_options_ = opendal_operator_options_new(); + if (!storage_config.access_key_id.empty() && + !storage_config.access_key_value.empty()) { + opendal_operator_options_set( + op_options_, "access_key_id", storage_config.access_key_id.c_str()); + opendal_operator_options_set(op_options_, + "secret_access_key", + storage_config.access_key_value.c_str()); + storageType = "s3"; + } + opendal_operator_options_set(op_options_, "root", "/"); + opendal_operator_options_set( + op_options_, "bucket", storage_config.bucket_name.c_str()); + opendal_operator_options_set(op_options_, + "region", + storage_config.region.empty() + ? "us-east-1" + : storage_config.region.c_str()); + opendal_operator_options_set( + op_options_, + "endpoint", + ((storage_config.useSSL ? "https://" : "http://") + + storage_config.address) + .c_str()); + opendal_operator_options_set( + op_options_, + "enable_virtual_host_style", + storage_config.useVirtualHost ? "true" : "false"); + + auto op = opendal_operator_new(storageType.c_str(), op_options_); + if (op.error != nullptr) { + THROWOPENDALERROR(op.error, "Init opendal error"); + } + op_ptr_ = op.op; + opendal_operator_options_free(op_options_); + LOG_SEGCORE_INFO_ << "init OpenDALChunkManager with parameter[storage: '" + << storageType << ", " << storage_config.cloud_provider + << "', endpoint: '" << storage_config.address + << "', default_bucket_name:'" + << storage_config.bucket_name << "', use_secure:'" + << std::boolalpha << storage_config.useSSL << "']"; +} + +OpenDALChunkManager::~OpenDALChunkManager() { + opendal_operator_free(op_ptr_); +} + +uint64_t +OpenDALChunkManager::Size(const std::string& filepath) { + auto ret = opendal_operator_stat(op_ptr_, filepath.c_str()); + if (ret.error != nullptr) { + THROWOPENDALERROR(ret.error, "GetObjectSize"); + } + auto size = opendal_metadata_content_length(ret.meta); + opendal_metadata_free(ret.meta); + return size; +} + +bool +OpenDALChunkManager::Exist(const std::string& filepath) { + auto ret = opendal_operator_is_exist(op_ptr_, filepath.c_str()); + if (ret.error != nullptr) { + THROWOPENDALERROR(ret.error, "ObjectExists"); + } + return ret.is_exist; +} + +void +OpenDALChunkManager::Remove(const std::string& filepath) { + auto ret = opendal_operator_delete(op_ptr_, filepath.c_str()); + if (ret != nullptr) { + THROWOPENDALERROR(ret, "RemoveObject"); + } +} + +std::vector +OpenDALChunkManager::ListWithPrefix(const std::string& filepath) { + auto ret = opendal_operator_list(op_ptr_, filepath.c_str()); + if (ret.error != nullptr) { + THROWOPENDALERROR(ret.error, "ListObjects"); + } + auto lister = OpendalLister(ret.lister); + std::vector objects; + opendal_result_lister_next result = opendal_lister_next(lister.Get()); + if (result.error != nullptr) { + THROWOPENDALERROR(result.error, "ListObjects"); + } + auto entry = result.entry; + while (entry) { + const char* de_path = opendal_entry_path(entry); + objects.push_back(std::string(de_path)); + opendal_entry_free(entry); + result = opendal_lister_next(lister.Get()); + if (result.error != nullptr) { + THROWOPENDALERROR(result.error, "ListObjects"); + } + entry = result.entry; + } + return objects; +} + +uint64_t +OpenDALChunkManager::Read(const std::string& filepath, + void* buf, + uint64_t size) { + auto ret = opendal_operator_reader(op_ptr_, filepath.c_str()); + if (ret.error != nullptr) { + THROWOPENDALERROR(ret.error, "GetObjectBuffer"); + } + auto reader = OpendalReader(ret.reader); + uint64_t buf_size = 16 * 1024; + uint64_t buf_index = 0; + while (true) { + auto read_ret = + opendal_reader_read(reader.Get(), + reinterpret_cast(buf) + buf_index, + buf_size); + buf_index += read_ret.size; + if (read_ret.error != nullptr) { + THROWOPENDALERROR(read_ret.error, "GetObjectBuffer"); + } + if (read_ret.size == 0) { + break; + } + } + if (buf_index != size) { + throw SegcoreError( + S3Error, + fmt::format( + "Read size mismatch, target size is {}, actual size is {}", + size, + buf_index)); + } + return buf_index; +} + +void +OpenDALChunkManager::Write(const std::string& filepath, + void* buf, + uint64_t size) { + auto ret = opendal_operator_write( + op_ptr_, filepath.c_str(), {reinterpret_cast(buf), size}); + if (ret != nullptr) { + THROWOPENDALERROR(ret, "Write"); + } +} + +} // namespace milvus::storage diff --git a/internal/core/src/storage/OpenDALChunkManager.h b/internal/core/src/storage/OpenDALChunkManager.h new file mode 100644 index 0000000000..30deb922c6 --- /dev/null +++ b/internal/core/src/storage/OpenDALChunkManager.h @@ -0,0 +1,106 @@ +#pragma once + +#include +#include +#include +#include + +#include "common/EasyAssert.h" +#include "storage/ChunkManager.h" +#include "storage/Types.h" +#include "opendal.h" + +namespace milvus::storage { +class OpenDALChunkManager : public ChunkManager { + public: + OpenDALChunkManager() = default; + explicit OpenDALChunkManager(const StorageConfig& storage_config); + + OpenDALChunkManager(const OpenDALChunkManager&); + OpenDALChunkManager& + operator=(const OpenDALChunkManager&); + + public: + virtual ~OpenDALChunkManager(); + + bool + Exist(const std::string& filepath) override; + + uint64_t + Size(const std::string& filepath) override; + + uint64_t + Read(const std::string& filepath, + uint64_t offset, + void* buf, + uint64_t len) override { + throw SegcoreError(NotImplemented, + GetName() + "Read with offset not implement"); + } + + void + Write(const std::string& filepath, + uint64_t offset, + void* buf, + uint64_t len) override { + throw SegcoreError(NotImplemented, + GetName() + "Write with offset not implement"); + } + + uint64_t + Read(const std::string& filepath, void* buf, uint64_t len) override; + + void + Write(const std::string& filepath, void* buf, uint64_t len) override; + + std::vector + ListWithPrefix(const std::string& filepath) override; + + void + Remove(const std::string& filepath) override; + + std::string + GetName() const override { + return "OpenDALChunkManager"; + } + + std::string + GetRootPath() const override { + return remote_root_path_; + } + + private: + std::string default_bucket_name_; + std::string remote_root_path_; + + const opendal_operator* op_ptr_; +}; +struct OpendalReader { + explicit OpendalReader(opendal_reader* reader) : reader_(reader) { + } + ~OpendalReader() { + opendal_reader_free(reader_); + } + opendal_reader* + Get() { + return reader_; + } + + private: + opendal_reader* reader_; +}; +struct OpendalLister { + explicit OpendalLister(opendal_lister* lister) : lister_(lister) { + } + ~OpendalLister() { + opendal_lister_free(lister_); + } + opendal_lister* + Get() { + return lister_; + } + + private: + opendal_lister* lister_; +}; +} // namespace milvus::storage diff --git a/internal/core/src/storage/RemoteChunkManagerSingleton.h b/internal/core/src/storage/RemoteChunkManagerSingleton.h index 75a070497c..7f8cded0cf 100644 --- a/internal/core/src/storage/RemoteChunkManagerSingleton.h +++ b/internal/core/src/storage/RemoteChunkManagerSingleton.h @@ -20,6 +20,7 @@ #include #include "storage/Util.h" +#include "opendal.h" namespace milvus::storage { diff --git a/internal/core/src/storage/Util.cpp b/internal/core/src/storage/Util.cpp index d5f41f8abb..7b762f4f8d 100644 --- a/internal/core/src/storage/Util.cpp +++ b/internal/core/src/storage/Util.cpp @@ -22,6 +22,7 @@ #include "common/EasyAssert.h" #include "common/Consts.h" #include "fmt/format.h" +#include "storage/ChunkManager.h" #ifdef AZURE_BUILD_DIR #include "storage/AzureChunkManager.h" #endif @@ -31,6 +32,7 @@ #include "storage/ThreadPools.h" #include "storage/LocalChunkManager.h" #include "storage/MinioChunkManager.h" +#include "storage/OpenDALChunkManager.h" #include "storage/MemFileManagerImpl.h" #include "storage/DiskFileManagerImpl.h" @@ -39,7 +41,8 @@ namespace milvus::storage { std::map ChunkManagerType_Map = { {"local", ChunkManagerType::Local}, {"minio", ChunkManagerType::Minio}, - {"remote", ChunkManagerType::Remote}}; + {"remote", ChunkManagerType::Remote}, + {"opendal", ChunkManagerType::OpenDAL}}; enum class CloudProviderType : int8_t { UNKNOWN = 0, @@ -448,45 +451,6 @@ EncodeAndUploadFieldSlice(ChunkManager* chunk_manager, return std::make_pair(std::move(object_key), serialized_index_size); } -// /** -// * Returns the current resident set size (physical memory use) measured -// * in bytes, or zero if the value cannot be determined on this OS. -// */ -// size_t -// getCurrentRSS() { -// #if defined(_WIN32) -// /* Windows -------------------------------------------------- */ -// PROCESS_MEMORY_COUNTERS info; -// GetProcessMemoryInfo(GetCurrentProcess(), &info, sizeof(info)); -// return (size_t)info.WorkingSetSize; - -// #elif defined(__APPLE__) && defined(__MACH__) -// /* OSX ------------------------------------------------------ */ -// struct mach_task_basic_info info; -// mach_msg_type_number_t infoCount = MACH_TASK_BASIC_INFO_COUNT; -// if (task_info(mach_task_self(), MACH_TASK_BASIC_INFO, (task_info_t)&info, &infoCount) != KERN_SUCCESS) -// return (size_t)0L; /* Can't access? */ -// return (size_t)info.resident_size; - -// #elif defined(__linux__) || defined(__linux) || defined(linux) || defined(__gnu_linux__) -// /* Linux ---------------------------------------------------- */ -// long rss = 0L; -// FILE* fp = NULL; -// if ((fp = fopen("/proc/self/statm", "r")) == NULL) -// return (size_t)0L; /* Can't open? */ -// if (fscanf(fp, "%*s%ld", &rss) != 1) { -// fclose(fp); -// return (size_t)0L; /* Can't read? */ -// } -// fclose(fp); -// return (size_t)rss * (size_t)sysconf(_SC_PAGESIZE); - -// #else -// /* AIX, BSD, Solaris, and Unknown OS ------------------------ */ -// return (size_t)0L; /* Unsupported. */ -// #endif -// } - std::vector GetObjectData(ChunkManager* remote_chunk_manager, const std::vector& remote_files) { @@ -608,6 +572,9 @@ CreateChunkManager(const StorageConfig& storage_config) { } } } + case ChunkManagerType::OpenDAL: { + return std::make_shared(storage_config); + } default: { PanicInfo(ConfigInvalid, diff --git a/internal/core/src/storage/storage_c.cpp b/internal/core/src/storage/storage_c.cpp index fb7d20879b..dd42256aad 100644 --- a/internal/core/src/storage/storage_c.cpp +++ b/internal/core/src/storage/storage_c.cpp @@ -66,6 +66,8 @@ InitRemoteChunkManagerSingleton(CStorageConfig c_storage_config) { std::string(c_storage_config.cloud_provider); storage_config.iam_endpoint = std::string(c_storage_config.iam_endpoint); + storage_config.cloud_provider = + std::string(c_storage_config.cloud_provider); storage_config.log_level = std::string(c_storage_config.log_level); storage_config.useSSL = c_storage_config.useSSL; storage_config.useIAM = c_storage_config.useIAM; diff --git a/internal/core/thirdparty/CMakeLists.txt b/internal/core/thirdparty/CMakeLists.txt index 98b628d541..0eb2de1d85 100644 --- a/internal/core/thirdparty/CMakeLists.txt +++ b/internal/core/thirdparty/CMakeLists.txt @@ -36,6 +36,7 @@ add_subdirectory(boost_ext) add_subdirectory(rocksdb) add_subdirectory(rdkafka) add_subdirectory(simdjson) +add_subdirectory(opendal) if (LINUX) add_subdirectory(jemalloc) diff --git a/internal/core/thirdparty/opendal/CMakeLists.txt b/internal/core/thirdparty/opendal/CMakeLists.txt new file mode 100644 index 0000000000..25fd05e83c --- /dev/null +++ b/internal/core/thirdparty/opendal/CMakeLists.txt @@ -0,0 +1,31 @@ +#------------------------------------------------------------------------------- +# Copyright (C) 2019-2020 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under the License. +#------------------------------------------------------------------------------- + +# ---------------------------------------------------------------------- +message(STATUS "Building (vendored) opendal from source") + +set(OPENDAL_LIB_DIR "${CMAKE_INSTALL_PREFIX}/lib") +set(OPENDAL_INCLUDE_DIR "${CMAKE_INSTALL_PREFIX}/include") +set(OPENDAL_NAME "libopendal_c${CMAKE_STATIC_LIBRARY_SUFFIX}") + +add_library(opendal STATIC IMPORTED) +set_target_properties(opendal + PROPERTIES + IMPORTED_GLOBAL TRUE + IMPORTED_LOCATION "${OPENDAL_LIB_DIR}/${OPENDAL_NAME}" + INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/include") + +get_target_property(OPENDAL_IMPORTED_LOCATION opendal IMPORTED_LOCATION) +get_target_property(OPENDAL_INTERFACE_INCLUDE_DIRECTORIES opendal INTERFACE_INCLUDE_DIRECTORIES) +message("OPENDAL_IMPORTED_LOCATION: ${OPENDAL_IMPORTED_LOCATION}") +message("OPENDAL_INTERFACE_INCLUDE_DIRECTORIES: ${OPENDAL_INTERFACE_INCLUDE_DIRECTORIES}") diff --git a/internal/core/unittest/test_sealed.cpp b/internal/core/unittest/test_sealed.cpp index f4b507d955..8c16643e1c 100644 --- a/internal/core/unittest/test_sealed.cpp +++ b/internal/core/unittest/test_sealed.cpp @@ -1181,7 +1181,7 @@ TEST(Sealed, GetVectorFromChunkCache) { auto sc = milvus::storage::StorageConfig{}; milvus::storage::RemoteChunkManagerSingleton::GetInstance().Init(sc); auto mcm = std::make_unique(sc); - mcm->CreateBucket(sc.bucket_name); + // mcm->CreateBucket(sc.bucket_name); milvus::storage::ChunkCacheSingleton::GetInstance().Init(mmap_dir, "willneed"); diff --git a/internal/storage/factory.go b/internal/storage/factory.go index 24a71cbf29..dd13fd3e59 100644 --- a/internal/storage/factory.go +++ b/internal/storage/factory.go @@ -48,7 +48,7 @@ func (f *ChunkManagerFactory) newChunkManager(ctx context.Context, engine string switch engine { case "local": return NewLocalChunkManager(RootPath(f.config.rootPath)), nil - case "minio": + case "minio", "opendal": return newMinioChunkManagerWithConfig(ctx, f.config) case "remote": return NewRemoteChunkManager(ctx, f.config) diff --git a/internal/util/indexcgowrapper/build_index_info.go b/internal/util/indexcgowrapper/build_index_info.go index ba24318202..18ed970818 100644 --- a/internal/util/indexcgowrapper/build_index_info.go +++ b/internal/util/indexcgowrapper/build_index_info.go @@ -48,18 +48,18 @@ func NewBuildIndexInfo(config *indexpb.StorageConfig) (*BuildIndexInfo, error) { cAccessValue := C.CString(config.SecretAccessKey) cRootPath := C.CString(config.RootPath) cStorageType := C.CString(config.StorageType) - cCloudProvider := C.CString(config.CloudProvider) cIamEndPoint := C.CString(config.IAMEndpoint) cRegion := C.CString(config.Region) + cCloudProvider := C.CString(config.CloudProvider) defer C.free(unsafe.Pointer(cAddress)) defer C.free(unsafe.Pointer(cBucketName)) defer C.free(unsafe.Pointer(cAccessKey)) defer C.free(unsafe.Pointer(cAccessValue)) defer C.free(unsafe.Pointer(cRootPath)) defer C.free(unsafe.Pointer(cStorageType)) - defer C.free(unsafe.Pointer(cCloudProvider)) defer C.free(unsafe.Pointer(cIamEndPoint)) defer C.free(unsafe.Pointer(cRegion)) + defer C.free(unsafe.Pointer(cCloudProvider)) storageConfig := C.CStorageConfig{ address: cAddress, bucket_name: cBucketName, @@ -67,8 +67,8 @@ func NewBuildIndexInfo(config *indexpb.StorageConfig) (*BuildIndexInfo, error) { access_key_value: cAccessValue, root_path: cRootPath, storage_type: cStorageType, - cloud_provider: cCloudProvider, iam_endpoint: cIamEndPoint, + cloud_provider: cCloudProvider, useSSL: C.bool(config.UseSSL), useIAM: C.bool(config.UseIAM), region: cRegion, diff --git a/internal/util/initcore/init_core.go b/internal/util/initcore/init_core.go index a9cd20d561..9bcbca2f66 100644 --- a/internal/util/initcore/init_core.go +++ b/internal/util/initcore/init_core.go @@ -62,8 +62,8 @@ func InitRemoteChunkManager(params *paramtable.ComponentParam) error { cAccessValue := C.CString(params.MinioCfg.SecretAccessKey.GetValue()) cRootPath := C.CString(params.MinioCfg.RootPath.GetValue()) cStorageType := C.CString(params.CommonCfg.StorageType.GetValue()) - cCloudProvider := C.CString(params.MinioCfg.CloudProvider.GetValue()) cIamEndPoint := C.CString(params.MinioCfg.IAMEndpoint.GetValue()) + cCloudProvider := C.CString(params.MinioCfg.CloudProvider.GetValue()) cLogLevel := C.CString(params.MinioCfg.LogLevel.GetValue()) cRegion := C.CString(params.MinioCfg.Region.GetValue()) defer C.free(unsafe.Pointer(cAddress)) @@ -72,10 +72,10 @@ func InitRemoteChunkManager(params *paramtable.ComponentParam) error { defer C.free(unsafe.Pointer(cAccessValue)) defer C.free(unsafe.Pointer(cRootPath)) defer C.free(unsafe.Pointer(cStorageType)) - defer C.free(unsafe.Pointer(cCloudProvider)) defer C.free(unsafe.Pointer(cIamEndPoint)) defer C.free(unsafe.Pointer(cLogLevel)) defer C.free(unsafe.Pointer(cRegion)) + defer C.free(unsafe.Pointer(cCloudProvider)) storageConfig := C.CStorageConfig{ address: cAddress, bucket_name: cBucketName, @@ -83,8 +83,8 @@ func InitRemoteChunkManager(params *paramtable.ComponentParam) error { access_key_value: cAccessValue, root_path: cRootPath, storage_type: cStorageType, - cloud_provider: cCloudProvider, iam_endpoint: cIamEndPoint, + cloud_provider: cCloudProvider, useSSL: C.bool(params.MinioCfg.UseSSL.GetAsBool()), useIAM: C.bool(params.MinioCfg.UseIAM.GetAsBool()), log_level: cLogLevel, diff --git a/pkg/util/paramtable/component_param.go b/pkg/util/paramtable/component_param.go index 11c9dd50e7..4dec480814 100644 --- a/pkg/util/paramtable/component_param.go +++ b/pkg/util/paramtable/component_param.go @@ -480,8 +480,8 @@ This configuration is only used by querynode and indexnode, it selects CPU instr p.StorageType = ParamItem{ Key: "common.storageType", Version: "2.0.0", - DefaultValue: "minio", - Doc: "please adjust in embedded Milvus: local", + DefaultValue: "opendal", + Doc: "please adjust in embedded Milvus: local, available values are [local, minio, remote, opendal]]", Export: true, } p.StorageType.Init(base.mgr) diff --git a/scripts/3rdparty_build.sh b/scripts/3rdparty_build.sh index 8b6a0554bb..f0d2e4167d 100644 --- a/scripts/3rdparty_build.sh +++ b/scripts/3rdparty_build.sh @@ -60,3 +60,25 @@ case "${unameOut}" in esac popd + +pushd ${ROOT_DIR}/cmake_build/thirdparty + +# git clone https://github.com/jiaoew1991/opendal.git opendal +git clone https://github.com/apache/incubator-opendal.git opendal +cd opendal +# git checkout blocking-layer +if command -v cargo >/dev/null 2>&1; then + echo "cargo exists" +else + bash -c "curl https://sh.rustup.rs -sSf | sh -s -- -y" || { echo 'rustup install failed'; exit 1;} + source $HOME/.cargo/env +fi +pushd bindings/c +cargo build || { echo 'opendal_c build failed'; exit 1; } +popd +mkdir -p ${ROOT_DIR}/internal/core/output/lib +mkdir -p ${ROOT_DIR}/internal/core/output/include +cp target/debug/libopendal_c.a ${ROOT_DIR}/internal/core/output/lib/libopendal_c.a +cp bindings/c/include/opendal.h ${ROOT_DIR}/internal/core/output/include/opendal.h + +popd diff --git a/scripts/run_cpp_codecov.sh b/scripts/run_cpp_codecov.sh index 3231062ba2..bd36863083 100755 --- a/scripts/run_cpp_codecov.sh +++ b/scripts/run_cpp_codecov.sh @@ -81,16 +81,17 @@ ${LCOV_CMD} -a ${FILE_INFO_BASE} -a ${FILE_INFO_UT} -o ${FILE_INFO_COMBINE} # remove unnecessary info ${LCOV_CMD} -r "${FILE_INFO_COMBINE}" -o "${FILE_INFO_OUTPUT}" \ "/usr/*" \ + "*/llvm/*" \ "*/src/pb/*" \ "*/src/core/bench/*" \ - "*/faiss_ep-prefix/*" \ - "*/boost/*" \ "*/unittest/*" \ - "*/thirdparty/*" + "*/thirdparty/*" \ + "*/3rdparty_download/*" \ + "*/.conan/data/*" # generate html report -#${LCOV_GEN_CMD} ${FILE_INFO_OUTPUT} --output-directory ${DIR_LCOV_OUTPUT}/ -#echo "Generate cpp code coverage report to ${DIR_LCOV_OUTPUT}" +${LCOV_GEN_CMD} ${FILE_INFO_OUTPUT} --output-directory ${DIR_LCOV_OUTPUT}/ +echo "Generate cpp code coverage report to ${DIR_LCOV_OUTPUT}" endTime=`date +%s`