admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

chunk manager supports using gcs & oss with accessKey (#25233)

Browse Source 
Signed-off-by: shaoyue.chen <shaoyue.chen@zilliz.com>
This commit is contained in:
shaoyue 2023-07-20 15:38:58 +08:00 committed by GitHub
parent 393932981c
commit 7bf39aed08
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 29 additions and 19 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

42
internal/core/src/storage/MinioChunkManager.cpp
View File

@ -89,7 +89,7 @@ ConvertFromAwsString(const Aws::String& aws_str) {
}
void
MinioChunkManager::InitSDKAPI(RemoteStorageType type) {
MinioChunkManager::InitSDKAPI(RemoteStorageType type, bool useIAM) {
std::scoped_lock lock{client_mutex_};
const size_t initCount = init_count_++;
if (initCount == 0) {
@ -103,9 +103,8 @@ MinioChunkManager::InitSDKAPI(RemoteStorageType type) {
sigemptyset(&psa.sa_mask);
sigaddset(&psa.sa_mask, SIGPIPE);
sigaction(SIGPIPE, &psa, 0);
if (type == RemoteStorageType::GOOGLE_CLOUD) {
if (type == RemoteStorageType::GOOGLE_CLOUD && useIAM) {
sdk_options_.httpOptions.httpClientFactory_create_fn = []() {
// auto credentials = google::cloud::oauth2_internal::GOOGLE_CLOUD_CPP_NS::GoogleDefaultCredentials();
auto credentials = std::make_shared<
google::cloud::oauth2_internal::GOOGLE_CLOUD_CPP_NS::
ComputeEngineCredentials>();
@ -149,21 +148,28 @@ MinioChunkManager::BuildS3Client(
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
false);
} else {
AssertInfo(!storage_config.access_key_id.empty(),
"if not use iam, access key should not be empty");
AssertInfo(!storage_config.access_key_value.empty(),
"if not use iam, access value should not be empty");
client_ = std::make_shared<Aws::S3::S3Client>(
Aws::Auth::AWSCredentials(
ConvertToAwsString(storage_config.access_key_id),
ConvertToAwsString(storage_config.access_key_value)),
config,
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
false);
BuildAccessKeyClient(storage_config, config);
}
}
void
MinioChunkManager::BuildAccessKeyClient(
const StorageConfig& storage_config,
const Aws::Client::ClientConfiguration& config) {
AssertInfo(!storage_config.access_key_id.empty(),
"if not use iam, access key should not be empty");
AssertInfo(!storage_config.access_key_value.empty(),
"if not use iam, access value should not be empty");
client_ = std::make_shared<Aws::S3::S3Client>(
Aws::Auth::AWSCredentials(
ConvertToAwsString(storage_config.access_key_id),
ConvertToAwsString(storage_config.access_key_value)),
config,
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
false);
}
void
MinioChunkManager::BuildAliyunCloudClient(
const StorageConfig& storage_config,
@ -185,7 +191,7 @@ MinioChunkManager::BuildAliyunCloudClient(
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
true);
} else {
throw std::runtime_error("aliyun cloud only support iam mode now");
BuildAccessKeyClient(storage_config, config);
}
}
@ -200,7 +206,7 @@ MinioChunkManager::BuildGoogleCloudClient(
Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never,
false);
} else {
throw std::runtime_error("google cloud only support iam mode now");
BuildAccessKeyClient(storage_config, config);
}
}
@ -216,7 +222,7 @@ MinioChunkManager::MinioChunkManager(const StorageConfig& storage_config)
storageType = RemoteStorageType::S3;
}
InitSDKAPI(storageType);
InitSDKAPI(storageType, storage_config.useIAM);
// The ClientConfiguration default constructor will take a long time.
// For more details, please refer to https://github.com/aws/aws-sdk-cpp/issues/1440

6
internal/core/src/storage/MinioChunkManager.h
View File

@ -149,7 +149,7 @@ class MinioChunkManager : public ChunkManager {
std::vector<std::string>
ListObjects(const char* bucket_name, const char* prefix = nullptr);
void
InitSDKAPI(RemoteStorageType type);
InitSDKAPI(RemoteStorageType type, bool useIAM);
void
ShutdownSDKAPI();
void
@ -163,6 +163,10 @@ class MinioChunkManager : public ChunkManager {
const Aws::Client::ClientConfiguration& config);
private:
void
BuildAccessKeyClient(const StorageConfig& storage_config,
const Aws::Client::ClientConfiguration& config);
Aws::SDKOptions sdk_options_;
static std::atomic<size_t> init_count_;
static std::mutex client_mutex_;