admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Make s3 client creation thread safe, prevent crashing (#20200)

Browse Source 
Signed-off-by: bigsheeper <yihao.dai@zilliz.com>

Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
This commit is contained in:
bigsheeper 2022-10-31 17:09:33 +08:00 committed by GitHub
parent 48f5c60070
commit 37dce8a660
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 24 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
internal/core/src/storage/MinioChunkManager.cpp
View File

@ -43,6 +43,7 @@
namespace milvus::storage {
std::atomic<size_t> MinioChunkManager::init_count_(0);
std::mutex MinioChunkManager::client_mutex_;
/**
* @brief convert std::string to Aws::String
@ -67,12 +68,27 @@ ConvertFromAwsString(const Aws::String& aws_str) {
return std::string(aws_str.c_str(), aws_str.size());
}
MinioChunkManager::MinioChunkManager(const StorageConfig& storage_config)
: default_bucket_name_(storage_config.bucket_name) {
void
MinioChunkManager::InitSDKAPI() {
std::scoped_lock lock{client_mutex_};
const size_t initCount = init_count_++;
if (initCount == 0) {
Aws::InitAPI(sdk_options_);
}
}
void
MinioChunkManager::ShutdownSDKAPI() {
std::scoped_lock lock{client_mutex_};
const size_t initCount = --init_count_;
if (initCount == 0) {
Aws::ShutdownAPI(sdk_options_);
}
}
MinioChunkManager::MinioChunkManager(const StorageConfig& storage_config)
: default_bucket_name_(storage_config.bucket_name) {
InitSDKAPI();
Aws::Client::ClientConfiguration config;
config.endpointOverride = ConvertToAwsString(storage_config.address);
@ -117,10 +133,7 @@ MinioChunkManager::MinioChunkManager(const StorageConfig& storage_config)
}
MinioChunkManager::~MinioChunkManager() {
const size_t initCount = --init_count_;
if (initCount == 0) {
Aws::ShutdownAPI(sdk_options_);
}
ShutdownSDKAPI();
client_.reset();
}

5
internal/core/src/storage/MinioChunkManager.h
View File

@ -112,10 +112,15 @@ class MinioChunkManager : public RemoteChunkManager {
GetObjectBuffer(const std::string& bucket_name, const std::string& object_name, void* buf, uint64_t size);
std::vector<std::string>
ListObjects(const char* bucket_name, const char* prefix = NULL);
void
InitSDKAPI();
void
ShutdownSDKAPI();
private:
const Aws::SDKOptions sdk_options_;
static std::atomic<size_t> init_count_;
static std::mutex client_mutex_;
std::shared_ptr<Aws::S3::S3Client> client_;
std::string default_bucket_name_;
};