diff --git a/go.mod b/go.mod index a86b7ddcb7..a77ed51b19 100644 --- a/go.mod +++ b/go.mod @@ -191,6 +191,7 @@ require ( github.com/streamnative/pulsarctl v0.5.0 // indirect github.com/stretchr/objx v0.5.0 // indirect github.com/subosito/gotenv v1.2.0 // indirect + github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.841 // indirect github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a // indirect github.com/tidwall/match v1.1.1 // indirect github.com/tidwall/pretty v1.2.0 // indirect diff --git a/go.sum b/go.sum index db13bafbca..d1f7f972df 100644 --- a/go.sum +++ b/go.sum @@ -287,6 +287,7 @@ github.com/go-latex/latex v0.0.0-20210118124228-b3d85cf34e07/go.mod h1:CO1AlKB2C github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -511,6 +512,7 @@ github.com/kataras/iris/v12 v12.1.8/go.mod h1:LMYy4VlP67TQ3Zgriz8RE2h2kMZV2SgMYb github.com/kataras/neffos v0.0.14/go.mod h1:8lqADm8PnbeFfL7CLXh1WHw53dG27MC3pgi2R1rmoTE= github.com/kataras/pio v0.0.2/go.mod h1:hAoW0t9UmXi4R5Oyq5Z4irTbaTsOemSrDGUtaTl7Dro= github.com/kataras/sitemap v0.0.5/go.mod h1:KY2eugMKiPwsJgx7+U103YZehfvNGOXURubcGyk0Bz8= +github.com/keybase/go-keychain v0.0.0-20190712205309-48d3d31d256d/go.mod h1:JJNrCn9otv/2QP4D7SMJBgaleKpOf66PnW6F5WGNRIc= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= @@ -842,6 +844,12 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/tencentcloud/tencentcloud-sdk-go v1.0.800/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4= +github.com/tencentcloud/tencentcloud-sdk-go v1.0.841/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4= +github.com/tencentcloud/tencentcloud-sdk-go v3.0.233+incompatible h1:q+D/Y9jla3afgsIihtyhwyl0c2W+eRWNM9ohVwPiiPw= +github.com/tencentcloud/tencentcloud-sdk-go v3.0.233+incompatible/go.mod h1:0PfYow01SHPMhKY31xa+EFz2RStxIqj6JFAJS+IkCi4= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.841 h1:SJJR4tLnr0V17uEVS+arAmR1yl8n6dObBZs77SAmXZE= +github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.841/go.mod h1:r5r4xbfxSaeR04b166HGsBa/R4U3SueirEUpXGuw+Q0= github.com/thoas/go-funk v0.9.1 h1:O549iLZqPpTUQ10ykd26sZhzD+rmR5pWhuElrhbC20M= github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4= github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM= diff --git a/internal/core/src/storage/CMakeLists.txt b/internal/core/src/storage/CMakeLists.txt index 4f6cc10ec7..eec8c8bcd8 100644 --- a/internal/core/src/storage/CMakeLists.txt +++ b/internal/core/src/storage/CMakeLists.txt @@ -56,7 +56,9 @@ set(STORAGE_FILES LocalChunkManager.cpp DiskFileManagerImpl.cpp ThreadPools.cpp - ChunkCache.cpp) + ChunkCache.cpp + TencentCloudCredentialsProvider.cpp + TencentCloudSTSClient.cpp) add_library(milvus_storage SHARED ${STORAGE_FILES}) diff --git a/internal/core/src/storage/ChunkManager.cpp b/internal/core/src/storage/ChunkManager.cpp index 27aa68e2a0..e4245c3d8e 100644 --- a/internal/core/src/storage/ChunkManager.cpp +++ b/internal/core/src/storage/ChunkManager.cpp @@ -30,7 +30,9 @@ #include "storage/MinioChunkManager.h" #include "storage/AliyunSTSClient.h" +#include "storage/TencentCloudSTSClient.h" #include "storage/AliyunCredentialsProvider.h" +#include "storage/TencentCloudCredentialsProvider.h" #include "common/Consts.h" #include "common/EasyAssert.h" #include "log/Log.h" @@ -188,4 +190,47 @@ AliyunChunkManager::AliyunChunkManager(const StorageConfig& storage_config) { storage_config.useSSL); } +TencentCloudChunkManager::TencentCloudChunkManager( + const StorageConfig& storage_config) { + default_bucket_name_ = storage_config.bucket_name; + remote_root_path_ = storage_config.root_path; + + InitSDKAPIDefault(storage_config.log_level); + + Aws::Client::ClientConfiguration config = generateConfig(storage_config); + + StorageConfig mutable_config = storage_config; + mutable_config.useVirtualHost = true; + if (storage_config.useIAM) { + auto tencent_cloud_provider = Aws::MakeShared< + Aws::Auth::TencentCloudSTSAssumeRoleWebIdentityCredentialsProvider>( + "TencentCloudSTSAssumeRoleWebIdentityCredentialsProvider"); + auto tencent_cloud_credentials = + tencent_cloud_provider->GetAWSCredentials(); + AssertInfo(!tencent_cloud_credentials.GetAWSAccessKeyId().empty(), + "if use iam, access key id should not be empty"); + AssertInfo(!tencent_cloud_credentials.GetAWSSecretKey().empty(), + "if use iam, secret key should not be empty"); + AssertInfo(!tencent_cloud_credentials.GetSessionToken().empty(), + "if use iam, token should not be empty"); + client_ = std::make_shared( + tencent_cloud_provider, + config, + Aws::Client::AWSAuthV4Signer::PayloadSigningPolicy::Never, + mutable_config.useVirtualHost); + } else { + BuildAccessKeyClient(mutable_config, config); + } + + PreCheck(storage_config); + + LOG_INFO( + "init TencentCloudChunkManager with " + "parameter[endpoint={}][bucket_name={}][root_path={}][use_secure={}]", + storage_config.address, + storage_config.bucket_name, + storage_config.root_path, + storage_config.useSSL); +} + } // namespace milvus::storage diff --git a/internal/core/src/storage/MinioChunkManager.cpp b/internal/core/src/storage/MinioChunkManager.cpp index 51e8def1e2..7c69f476f4 100644 --- a/internal/core/src/storage/MinioChunkManager.cpp +++ b/internal/core/src/storage/MinioChunkManager.cpp @@ -32,6 +32,8 @@ #include "storage/AliyunSTSClient.h" #include "storage/AliyunCredentialsProvider.h" +#include "storage/TencentCloudSTSClient.h" +#include "storage/TencentCloudCredentialsProvider.h" #include "storage/prometheus_client.h" #include "common/EasyAssert.h" #include "log/Log.h" diff --git a/internal/core/src/storage/MinioChunkManager.h b/internal/core/src/storage/MinioChunkManager.h index da839e64a7..4e2586e29b 100644 --- a/internal/core/src/storage/MinioChunkManager.h +++ b/internal/core/src/storage/MinioChunkManager.h @@ -45,7 +45,11 @@ namespace milvus::storage { -enum class RemoteStorageType { S3 = 0, GOOGLE_CLOUD = 1, ALIYUN_CLOUD = 2 }; +enum class RemoteStorageType { + S3 = 0, + GOOGLE_CLOUD = 1, + ALIYUN_CLOUD = 2, +}; template @@ -258,6 +262,15 @@ class AliyunChunkManager : public MinioChunkManager { } }; +class TencentCloudChunkManager : public MinioChunkManager { + public: + explicit TencentCloudChunkManager(const StorageConfig& storage_config); + virtual std::string + GetName() const { + return "TencentCloudChunkManager"; + } +}; + using MinioChunkManagerPtr = std::unique_ptr; static const char* GOOGLE_CLIENT_FACTORY_ALLOCATION_TAG = diff --git a/internal/core/src/storage/TencentCloudCredentialsProvider.cpp b/internal/core/src/storage/TencentCloudCredentialsProvider.cpp new file mode 100644 index 0000000000..88826eb29e --- /dev/null +++ b/internal/core/src/storage/TencentCloudCredentialsProvider.cpp @@ -0,0 +1,185 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include +#include +#include +#include +#include +#include +#include "TencentCloudCredentialsProvider.h" + +static const char STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG[] = + "TencentCloudSTSAssumeRoleWebIdentityCredentialsProvider"; +static const int STS_CREDENTIAL_PROVIDER_EXPIRATION_GRACE_PERIOD = + 7200; // tencent cloud support 7200s. + +namespace Aws { +namespace Auth { +TencentCloudSTSAssumeRoleWebIdentityCredentialsProvider:: + TencentCloudSTSAssumeRoleWebIdentityCredentialsProvider() + : m_initialized(false) { + m_region = Aws::Environment::GetEnv("TKE_REGION"); + m_roleArn = Aws::Environment::GetEnv("TKE_ROLE_ARN"); + m_tokenFile = Aws::Environment::GetEnv("TKE_WEB_IDENTITY_TOKEN_FILE"); + m_providerId = Aws::Environment::GetEnv("TKE_PROVIDER_ID"); + auto currentTimePoint = std::chrono::high_resolution_clock::now(); + auto nanoseconds = std::chrono::time_point_cast( + currentTimePoint); + auto timestamp = nanoseconds.time_since_epoch().count(); + m_sessionName = "tencentcloud-cpp-sdk-" + std::to_string(timestamp / 1000); + + if (m_roleArn.empty() || m_tokenFile.empty() || m_region.empty()) { + auto profile = Aws::Config::GetCachedConfigProfile( + Aws::Auth::GetConfigProfileName()); + m_roleArn = profile.GetRoleArn(); + m_tokenFile = profile.GetValue("web_identity_token_file"); + m_sessionName = profile.GetValue("role_session_name"); + } + + if (m_tokenFile.empty()) { + AWS_LOGSTREAM_WARN(STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG, + "Token file must be specified to use STS AssumeRole " + "web identity creds provider."); + return; // No need to do further constructing + } else { + AWS_LOGSTREAM_DEBUG(STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG, + "Resolved token_file from profile_config or " + "environment variable to be " + << m_tokenFile); + } + + if (m_roleArn.empty()) { + AWS_LOGSTREAM_WARN(STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG, + "RoleArn must be specified to use STS AssumeRole " + "web identity creds provider."); + return; // No need to do further constructing + } else { + AWS_LOGSTREAM_DEBUG(STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG, + "Resolved role_arn from profile_config or " + "environment variable to be " + << m_roleArn); + } + + if (m_region.empty()) { + AWS_LOGSTREAM_WARN(STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG, + "Region must be specified to use STS AssumeRole " + "web identity creds provider."); + return; // No need to do further constructing + } else { + AWS_LOGSTREAM_DEBUG(STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG, + "Resolved region from profile_config or " + "environment variable to be " + << m_region); + } + + if (m_sessionName.empty()) { + m_sessionName = Aws::Utils::UUID::RandomUUID(); + } else { + AWS_LOGSTREAM_DEBUG(STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG, + "Resolved session_name from profile_config or " + "environment variable to be " + << m_sessionName); + } + + Aws::Client::ClientConfiguration config; + config.scheme = Aws::Http::Scheme::HTTPS; + config.region = m_region; + + Aws::Vector retryableErrors; + retryableErrors.push_back("IDPCommunicationError"); + retryableErrors.push_back("InvalidIdentityToken"); + + config.retryStrategy = + Aws::MakeShared( + STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG, + retryableErrors, + 3 /*maxRetries*/); + + m_client = Aws::MakeUnique( + STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG, config); + m_initialized = true; + AWS_LOGSTREAM_INFO( + STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG, + "Creating STS AssumeRole with web identity creds provider."); +} + +Aws::Auth::AWSCredentials +TencentCloudSTSAssumeRoleWebIdentityCredentialsProvider::GetAWSCredentials() { + // A valid client means required information like role arn and token file were constructed correctly. + // We can use this provider to load creds, otherwise, we can just return empty creds. + if (!m_initialized) { + return Aws::Auth::AWSCredentials(); + } + RefreshIfExpired(); + Aws::Utils::Threading::ReaderLockGuard guard(m_reloadLock); + return m_credentials; +} + +void +TencentCloudSTSAssumeRoleWebIdentityCredentialsProvider::Reload() { + AWS_LOGSTREAM_INFO( + STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG, + "Credentials have expired, attempting to renew from STS."); + + Aws::IFStream tokenFile(m_tokenFile.c_str()); + if (tokenFile) { + Aws::String token((std::istreambuf_iterator(tokenFile)), + std::istreambuf_iterator()); + if (!token.empty() && token.back() == '\n') { + token.pop_back(); + } + m_token = token; + } else { + AWS_LOGSTREAM_ERROR(STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG, + "Can't open token file: " << m_tokenFile); + return; + } + Aws::Internal::TencentCloudSTSCredentialsClient:: + STSAssumeRoleWithWebIdentityRequest request{ + m_region, m_providerId, m_token, m_roleArn, m_sessionName}; + + auto result = m_client->GetAssumeRoleWithWebIdentityCredentials(request); + AWS_LOGSTREAM_TRACE( + STS_ASSUME_ROLE_WEB_IDENTITY_LOG_TAG, + "Successfully retrieved credentials with AWS_ACCESS_KEY: " + << result.creds.GetAWSAccessKeyId()); + m_credentials = result.creds; +} + +bool +TencentCloudSTSAssumeRoleWebIdentityCredentialsProvider::ExpiresSoon() const { + return ( + (m_credentials.GetExpiration() - Aws::Utils::DateTime::Now()).count() < + STS_CREDENTIAL_PROVIDER_EXPIRATION_GRACE_PERIOD); +} + +void +TencentCloudSTSAssumeRoleWebIdentityCredentialsProvider::RefreshIfExpired() { + Aws::Utils::Threading::ReaderLockGuard guard(m_reloadLock); + if (!m_credentials.IsEmpty() && !ExpiresSoon()) { + return; + } + + guard.UpgradeToWriterLock(); + if (!m_credentials.IsExpiredOrEmpty() && !ExpiresSoon()) { + return; + } + + Reload(); +} +} // namespace Auth +}; // namespace Aws diff --git a/internal/core/src/storage/TencentCloudCredentialsProvider.h b/internal/core/src/storage/TencentCloudCredentialsProvider.h new file mode 100644 index 0000000000..c3314cd9b8 --- /dev/null +++ b/internal/core/src/storage/TencentCloudCredentialsProvider.h @@ -0,0 +1,68 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include "TencentCloudSTSClient.h" + +namespace Aws { +namespace Auth { +/** + * To support retrieving credentials of STS AssumeRole with web identity. + * Note that STS accepts request with protocol of queryxml. Calling GetAWSCredentials() will trigger (if expired) + * a query request using AWSHttpResourceClient under the hood. + */ +class AWS_CORE_API TencentCloudSTSAssumeRoleWebIdentityCredentialsProvider + : public AWSCredentialsProvider { + public: + TencentCloudSTSAssumeRoleWebIdentityCredentialsProvider(); + + /** + * Retrieves the credentials if found, otherwise returns empty credential set. + */ + AWSCredentials + GetAWSCredentials() override; + + protected: + void + Reload() override; + + private: + void + RefreshIfExpired(); + Aws::String + CalculateQueryString() const; + + Aws::UniquePtr m_client; + Aws::Auth::AWSCredentials m_credentials; + Aws::String m_region; + Aws::String m_roleArn; + Aws::String m_tokenFile; + Aws::String m_sessionName; + Aws::String m_providerId; + Aws::String m_token; + bool m_initialized; + bool + ExpiresSoon() const; +}; +} // namespace Auth +} // namespace Aws diff --git a/internal/core/src/storage/TencentCloudSTSClient.cpp b/internal/core/src/storage/TencentCloudSTSClient.cpp new file mode 100644 index 0000000000..18915c2bb4 --- /dev/null +++ b/internal/core/src/storage/TencentCloudSTSClient.cpp @@ -0,0 +1,150 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include "TencentCloudSTSClient.h" + +namespace Aws { +namespace Http { +class HttpClient; +class HttpRequest; +enum class HttpResponseCode; +} // namespace Http + +namespace Client { +Aws::String +ComputeUserAgentString(); +} + +namespace Internal { + +static const char STS_RESOURCE_CLIENT_LOG_TAG[] = + "TencentCloudSTSResourceClient"; // [tencent cloud] + +TencentCloudSTSCredentialsClient::TencentCloudSTSCredentialsClient( + const Aws::Client::ClientConfiguration& clientConfiguration) + : AWSHttpResourceClient(clientConfiguration, STS_RESOURCE_CLIENT_LOG_TAG) { + SetErrorMarshaller(Aws::MakeUnique( + STS_RESOURCE_CLIENT_LOG_TAG)); + + // [tencent cloud] + m_endpoint = "https://sts.tencentcloudapi.com"; + + AWS_LOGSTREAM_INFO( + STS_RESOURCE_CLIENT_LOG_TAG, + "Creating STS ResourceClient with endpoint: " << m_endpoint); +} + +TencentCloudSTSCredentialsClient::STSAssumeRoleWithWebIdentityResult +TencentCloudSTSCredentialsClient::GetAssumeRoleWithWebIdentityCredentials( + const STSAssumeRoleWithWebIdentityRequest& request) { + // Calculate query string + Aws::StringStream ss; + // curl -X POST "https://sts.tencentcloudapi.com" + // -d "{\"ProviderId\": $ProviderId, \"WebIdentityToken\": $WebIdentityToken,\"RoleArn\":$RoleArn,\"RoleSessionName\":$RoleSessionName,\"DurationSeconds\":7200}" + // -H "Authorization: SKIP" + // -H "Content-Type: application/json; charset=utf-8" + // -H "Host: sts.tencentcloudapi.com" + // -H "X-TC-Action: AssumeRoleWithWebIdentity" + // -H "X-TC-Timestamp: $timestamp" + // -H "X-TC-Version: 2018-08-13" + // -H "X-TC-Region: $region" + // -H "X-TC-Token: $token" + + ss << R"({"ProviderId": ")" << request.providerId + << R"(", "WebIdentityToken": ")" << request.webIdentityToken + << R"(", "RoleArn": ")" << request.roleArn + << R"(", "RoleSessionName": ")" << request.roleSessionName << R"("})"; + + std::shared_ptr httpRequest( + Aws::Http::CreateHttpRequest( + m_endpoint, + Aws::Http::HttpMethod::HTTP_POST, + Aws::Utils::Stream::DefaultResponseStreamFactoryMethod)); + + httpRequest->SetUserAgent(Aws::Client::ComputeUserAgentString()); + httpRequest->SetHeaderValue("Authorization", "SKIP"); + httpRequest->SetHeaderValue("Host", "sts.tencentcloudapi.com"); + httpRequest->SetHeaderValue("X-TC-Action", "AssumeRoleWithWebIdentity"); + httpRequest->SetHeaderValue( + "X-TC-Timestamp", + std::to_string(Aws::Utils::DateTime::Now().Seconds())); + httpRequest->SetHeaderValue("X-TC-Version", "2018-08-13"); + httpRequest->SetHeaderValue("X-TC-Region", request.region); + httpRequest->SetHeaderValue("X-TC-Token", ""); + + std::shared_ptr body = + Aws::MakeShared("STS_RESOURCE_CLIENT_LOG_TAG"); + *body << ss.str(); + + httpRequest->AddContentBody(body); + body->seekg(0, body->end); + auto streamSize = body->tellg(); + body->seekg(0, body->beg); + Aws::StringStream contentLength; + contentLength << streamSize; + httpRequest->SetContentLength(contentLength.str()); + // httpRequest->SetContentType("application/x-www-form-urlencoded"); + httpRequest->SetContentType("application/json; charset=utf-8"); + + auto headers = httpRequest->GetHeaders(); + Aws::String credentialsStr = + GetResourceWithAWSWebServiceResult(httpRequest).GetPayload(); + + // Parse credentials + STSAssumeRoleWithWebIdentityResult result; + if (credentialsStr.empty()) { + AWS_LOGSTREAM_WARN(STS_RESOURCE_CLIENT_LOG_TAG, + "Get an empty credential from sts"); + return result; + } + + auto json = Utils::Json::JsonView(credentialsStr); + auto rootNode = json.GetObject("Response"); + if (rootNode.IsNull()) { + AWS_LOGSTREAM_WARN(STS_RESOURCE_CLIENT_LOG_TAG, + "Get Response from credential result failed"); + return result; + } + + auto credentialsNode = rootNode.GetObject("Credentials"); + if (credentialsNode.IsNull()) { + AWS_LOGSTREAM_WARN(STS_RESOURCE_CLIENT_LOG_TAG, + "Get Credentials from Response failed"); + return result; + } + result.creds.SetAWSAccessKeyId(credentialsNode.GetString("TmpSecretId")); + result.creds.SetAWSSecretKey(credentialsNode.GetString("TmpSecretKey")); + result.creds.SetSessionToken(credentialsNode.GetString("Token")); + result.creds.SetExpiration(Aws::Utils::DateTime( + Aws::Utils::StringUtils::Trim(rootNode.GetString("Expiration").c_str()) + .c_str(), + Aws::Utils::DateFormat::ISO_8601)); + + return result; +} +} // namespace Internal +} // namespace Aws diff --git a/internal/core/src/storage/TencentCloudSTSClient.h b/internal/core/src/storage/TencentCloudSTSClient.h new file mode 100644 index 0000000000..2cf7e2b9f7 --- /dev/null +++ b/internal/core/src/storage/TencentCloudSTSClient.h @@ -0,0 +1,85 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License + +/** + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace Aws { +namespace Http { +class HttpClient; +class HttpRequest; +enum class HttpResponseCode; +} // namespace Http + +namespace Internal { +/** + * To support retrieving credentials from STS. + * Note that STS accepts request with protocol of queryxml. Calling GetResource() will trigger + * a query request using AWSHttpResourceClient under the hood. + */ +class AWS_CORE_API TencentCloudSTSCredentialsClient + : public AWSHttpResourceClient { + public: + /** + * Initializes the provider to retrieve credentials from STS when it expires. + */ + explicit TencentCloudSTSCredentialsClient( + const Client::ClientConfiguration& clientConfiguration); + + TencentCloudSTSCredentialsClient& + operator=(TencentCloudSTSCredentialsClient& rhs) = delete; + TencentCloudSTSCredentialsClient( + const TencentCloudSTSCredentialsClient& rhs) = delete; + TencentCloudSTSCredentialsClient& + operator=(TencentCloudSTSCredentialsClient&& rhs) = delete; + TencentCloudSTSCredentialsClient( + const TencentCloudSTSCredentialsClient&& rhs) = delete; + + // If you want to make an AssumeRoleWithWebIdentity call to sts. use these classes to pass data to and get info from + // TencentCloudSTSCredentialsClient client. If you want to make an AssumeRole call to sts, define the request/result + // members class/struct like this. + struct STSAssumeRoleWithWebIdentityRequest { + Aws::String region; + Aws::String providerId; + Aws::String webIdentityToken; + Aws::String roleArn; + Aws::String roleSessionName; + }; + + struct STSAssumeRoleWithWebIdentityResult { + Aws::Auth::AWSCredentials creds; + }; + + STSAssumeRoleWithWebIdentityResult + GetAssumeRoleWithWebIdentityCredentials( + const STSAssumeRoleWithWebIdentityRequest& request); + + private: + Aws::String m_endpoint; +}; +} // namespace Internal +} // namespace Aws diff --git a/internal/core/src/storage/Util.cpp b/internal/core/src/storage/Util.cpp index 6038e55e0e..ca171ea466 100644 --- a/internal/core/src/storage/Util.cpp +++ b/internal/core/src/storage/Util.cpp @@ -54,13 +54,15 @@ enum class CloudProviderType : int8_t { GCP = 2, ALIYUN = 3, AZURE = 4, + TENCENTCLOUD = 5, }; std::map CloudProviderType_Map = { {"aws", CloudProviderType::AWS}, {"gcp", CloudProviderType::GCP}, {"aliyun", CloudProviderType::ALIYUN}, - {"azure", CloudProviderType::AZURE}}; + {"azure", CloudProviderType::AZURE}, + {"tencent", CloudProviderType::TENCENTCLOUD}}; std::map ReadAheadPolicy_Map = { {"normal", MADV_NORMAL}, @@ -670,6 +672,10 @@ CreateChunkManager(const StorageConfig& storage_config) { case CloudProviderType::ALIYUN: { return std::make_shared(storage_config); } + case CloudProviderType::TENCENTCLOUD: { + return std::make_shared( + storage_config); + } #ifdef AZURE_BUILD_DIR case CloudProviderType::AZURE: { return std::make_shared(storage_config); diff --git a/internal/core/unittest/CMakeLists.txt b/internal/core/unittest/CMakeLists.txt index bf50f257ef..f4cb55e168 100644 --- a/internal/core/unittest/CMakeLists.txt +++ b/internal/core/unittest/CMakeLists.txt @@ -84,7 +84,7 @@ if (DEFINED AZURE_BUILD_DIR) set(MILVUS_TEST_FILES ${MILVUS_TEST_FILES} test_azure_chunk_manager.cpp - #need update aws-sdk-cpp, see more from https://github.com/aws/aws-sdk-cpp/issues/2119 + #need update aws-sdk-cpp, see more from https://github.com/aws/aws-sdk-cpp/issues/2119 #test_remote_chunk_manager.cpp ) include_directories("${AZURE_BUILD_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}/include") diff --git a/internal/storage/minio_object_storage.go b/internal/storage/minio_object_storage.go index bf273ec99d..aa52255bbc 100644 --- a/internal/storage/minio_object_storage.go +++ b/internal/storage/minio_object_storage.go @@ -30,6 +30,7 @@ import ( "github.com/milvus-io/milvus/internal/storage/aliyun" "github.com/milvus-io/milvus/internal/storage/gcp" + "github.com/milvus-io/milvus/internal/storage/tencent" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/util/retry" ) @@ -62,6 +63,12 @@ func newMinioClient(ctx context.Context, c *config) (*minio.Client, error) { if !c.useIAM { creds = credentials.NewStaticV2(c.accessKeyID, c.secretAccessKeyID, "") } + case CloudProviderTencent: + newMinioFn = tencent.NewMinioClient + if !c.useIAM { + creds = credentials.NewStaticV4(c.accessKeyID, c.secretAccessKeyID, "") + } + default: // aws, minio matchedDefault = true } diff --git a/internal/storage/remote_chunk_manager.go b/internal/storage/remote_chunk_manager.go index ecd05f53b6..48f098fbfa 100644 --- a/internal/storage/remote_chunk_manager.go +++ b/internal/storage/remote_chunk_manager.go @@ -38,11 +38,11 @@ import ( ) const ( - CloudProviderGCP = "gcp" - CloudProviderAWS = "aws" - CloudProviderAliyun = "aliyun" - - CloudProviderAzure = "azure" + CloudProviderGCP = "gcp" + CloudProviderAWS = "aws" + CloudProviderAliyun = "aliyun" + CloudProviderAzure = "azure" + CloudProviderTencent = "tencent" ) type ObjectStorage interface { diff --git a/internal/storage/tencent/tencent.go b/internal/storage/tencent/tencent.go new file mode 100644 index 0000000000..bd0f727630 --- /dev/null +++ b/internal/storage/tencent/tencent.go @@ -0,0 +1,85 @@ +package tencent + +import ( + "fmt" + + "github.com/cockroachdb/errors" + "github.com/minio/minio-go/v7" + minioCred "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common" +) + +// NewMinioClient returns a minio.Client which is compatible for tencent OSS +func NewMinioClient(address string, opts *minio.Options) (*minio.Client, error) { + if opts == nil { + opts = &minio.Options{} + } + if opts.Creds == nil { + credProvider, err := NewCredentialProvider() + if err != nil { + return nil, errors.Wrap(err, "failed to create credential provider") + } + opts.Creds = minioCred.New(credProvider) + } + if address == "" { + address = fmt.Sprintf("cos.%s.myqcloud.com", opts.Region) + opts.Secure = true + } + return minio.New(address, opts) +} + +// Credential is defined to mock tencent credential.Credentials +// +//go:generate mockery --name=Credential --with-expecter +type Credential interface { + common.CredentialIface +} + +// CredentialProvider implements "github.com/minio/minio-go/v7/pkg/credentials".Provider +// also implements transport +type CredentialProvider struct { + // tencentCreds doesn't provide a way to get the expired time, so we use the cache to check if it's expired + // when tencentCreds.GetSecretId is different from the cache, we know it's expired + akCache string + tencentCreds Credential +} + +func NewCredentialProvider() (minioCred.Provider, error) { + provider, err := common.DefaultTkeOIDCRoleArnProvider() + if err != nil { + return nil, errors.Wrap(err, "failed to create tencent credential provider") + } + + cred, err := provider.GetCredential() + if err != nil { + return nil, errors.Wrap(err, "failed to get tencent credential") + } + return &CredentialProvider{tencentCreds: cred}, nil +} + +// Retrieve returns nil if it successfully retrieved the value. +// Error is returned if the value were not obtainable, or empty. +// according to the caller minioCred.Credentials.Get(), +// it already has a lock, so we don't need to worry about concurrency +func (c *CredentialProvider) Retrieve() (minioCred.Value, error) { + ret := minioCred.Value{} + ak := c.tencentCreds.GetSecretId() + ret.AccessKeyID = ak + c.akCache = ak + + sk := c.tencentCreds.GetSecretKey() + ret.SecretAccessKey = sk + + securityToken := c.tencentCreds.GetToken() + ret.SessionToken = securityToken + return ret, nil +} + +// IsExpired returns if the credentials are no longer valid, and need +// to be retrieved. +// according to the caller minioCred.Credentials.IsExpired(), +// it already has a lock, so we don't need to worry about concurrency +func (c CredentialProvider) IsExpired() bool { + ak := c.tencentCreds.GetSecretId() + return ak != c.akCache +} diff --git a/internal/storage/tencent/tencent_test.go b/internal/storage/tencent/tencent_test.go new file mode 100644 index 0000000000..78526fc3a7 --- /dev/null +++ b/internal/storage/tencent/tencent_test.go @@ -0,0 +1,25 @@ +package tencent + +import ( + "testing" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/stretchr/testify/assert" +) + +func Test_NewMinioClient(t *testing.T) { + t.Run("ak sk ok", func(t *testing.T) { + minioCli, err := NewMinioClient("xxx.cos.ap-beijing.myqcloud.com", &minio.Options{ + Creds: credentials.NewStaticV2("ak", "sk", ""), + Secure: true, + }) + assert.NoError(t, err) + assert.Equal(t, "https", minioCli.EndpointURL().Scheme) + }) + + t.Run("iam failed", func(t *testing.T) { + _, err := NewMinioClient("", nil) + assert.Error(t, err) + }) +}