From f455910bee8354b996e1580f2fcb7b11321da191 Mon Sep 17 00:00:00 2001 From: Xiaofan <83447078+xiaofan-luan@users.noreply.github.com> Date: Thu, 27 Nov 2025 14:29:07 +0800 Subject: [PATCH] fix: support azure blob storage with federated token (#45632) fix #44582 related to #44583 Co-authored-by: DuMinhLe Signed-off-by: xiaofanluan --- internal/core/src/storage/CMakeLists.txt | 5 ++ internal/core/src/storage/ChunkManager.cpp | 12 ++-- .../src/storage/RemoteChunkManagerTest.cpp | 2 +- internal/core/src/storage/Util.cpp | 2 +- .../AliyunCredentialsProvider.cpp | 0 .../{ => aliyun}/AliyunCredentialsProvider.h | 0 .../storage/{ => aliyun}/AliyunSTSClient.cpp | 0 .../storage/{ => aliyun}/AliyunSTSClient.h | 0 .../core/src/storage/aliyun/CMakeLists.txt | 19 ++++++ .../storage/azure/AzureBlobChunkManager.cpp | 60 +++++++++++++------ .../core/src/storage/huawei/CMakeLists.txt | 19 ++++++ .../HuaweiCloudCredentialsProvider.cpp | 0 .../HuaweiCloudCredentialsProvider.h | 0 .../{ => huawei}/HuaweiCloudSTSClient.cpp | 0 .../{ => huawei}/HuaweiCloudSTSClient.h | 0 .../core/src/storage/minio/CMakeLists.txt | 18 ++++++ .../storage/{ => minio}/MinioChunkManager.cpp | 10 ++-- .../storage/{ => minio}/MinioChunkManager.h | 0 .../{ => minio}/MinioChunkManagerTest.cpp | 2 +- .../core/src/storage/tencent/CMakeLists.txt | 19 ++++++ .../TencentCloudCredentialsProvider.cpp | 0 .../TencentCloudCredentialsProvider.h | 0 .../{ => tencent}/TencentCloudSTSClient.cpp | 0 .../{ => tencent}/TencentCloudSTSClient.h | 0 internal/proxy/task_index.go | 2 +- pkg/objectstorage/util.go | 25 ++++++-- 26 files changed, 157 insertions(+), 38 deletions(-) rename internal/core/src/storage/{ => aliyun}/AliyunCredentialsProvider.cpp (100%) rename internal/core/src/storage/{ => aliyun}/AliyunCredentialsProvider.h (100%) rename internal/core/src/storage/{ => aliyun}/AliyunSTSClient.cpp (100%) rename internal/core/src/storage/{ => aliyun}/AliyunSTSClient.h (100%) create mode 100644 internal/core/src/storage/aliyun/CMakeLists.txt create mode 100644 internal/core/src/storage/huawei/CMakeLists.txt rename internal/core/src/storage/{ => huawei}/HuaweiCloudCredentialsProvider.cpp (100%) rename internal/core/src/storage/{ => huawei}/HuaweiCloudCredentialsProvider.h (100%) rename internal/core/src/storage/{ => huawei}/HuaweiCloudSTSClient.cpp (100%) rename internal/core/src/storage/{ => huawei}/HuaweiCloudSTSClient.h (100%) create mode 100644 internal/core/src/storage/minio/CMakeLists.txt rename internal/core/src/storage/{ => minio}/MinioChunkManager.cpp (99%) rename internal/core/src/storage/{ => minio}/MinioChunkManager.h (100%) rename internal/core/src/storage/{ => minio}/MinioChunkManagerTest.cpp (99%) create mode 100644 internal/core/src/storage/tencent/CMakeLists.txt rename internal/core/src/storage/{ => tencent}/TencentCloudCredentialsProvider.cpp (100%) rename internal/core/src/storage/{ => tencent}/TencentCloudCredentialsProvider.h (100%) rename internal/core/src/storage/{ => tencent}/TencentCloudSTSClient.cpp (100%) rename internal/core/src/storage/{ => tencent}/TencentCloudSTSClient.h (100%) diff --git a/internal/core/src/storage/CMakeLists.txt b/internal/core/src/storage/CMakeLists.txt index de44b81199..5d1ed71c8b 100644 --- a/internal/core/src/storage/CMakeLists.txt +++ b/internal/core/src/storage/CMakeLists.txt @@ -16,6 +16,11 @@ add_source_at_current_directory() +add_subdirectory(minio) +add_subdirectory(aliyun) +add_subdirectory(tencent) +add_subdirectory(huawei) + if (ENABLE_GCP_NATIVE) add_definitions(-DENABLE_GCP_NATIVE) add_subdirectory(gcp-native-storage) diff --git a/internal/core/src/storage/ChunkManager.cpp b/internal/core/src/storage/ChunkManager.cpp index d013b4a7a6..be87354f7b 100644 --- a/internal/core/src/storage/ChunkManager.cpp +++ b/internal/core/src/storage/ChunkManager.cpp @@ -28,12 +28,12 @@ #include #include -#include "storage/MinioChunkManager.h" -#include "storage/AliyunSTSClient.h" -#include "storage/TencentCloudSTSClient.h" -#include "storage/AliyunCredentialsProvider.h" -#include "storage/TencentCloudCredentialsProvider.h" -#include "storage/HuaweiCloudCredentialsProvider.h" +#include "storage/minio/MinioChunkManager.h" +#include "storage/aliyun/AliyunSTSClient.h" +#include "storage/aliyun/AliyunCredentialsProvider.h" +#include "storage/tencent/TencentCloudSTSClient.h" +#include "storage/tencent/TencentCloudCredentialsProvider.h" +#include "storage/huawei/HuaweiCloudCredentialsProvider.h" #include "common/Consts.h" #include "common/EasyAssert.h" #include "log/Log.h" diff --git a/internal/core/src/storage/RemoteChunkManagerTest.cpp b/internal/core/src/storage/RemoteChunkManagerTest.cpp index b9d58d2348..43650967b3 100644 --- a/internal/core/src/storage/RemoteChunkManagerTest.cpp +++ b/internal/core/src/storage/RemoteChunkManagerTest.cpp @@ -13,7 +13,7 @@ #include #include -#include "storage/MinioChunkManager.h" +#include "storage/minio/MinioChunkManager.h" #include "storage/Util.h" using namespace std; diff --git a/internal/core/src/storage/Util.cpp b/internal/core/src/storage/Util.cpp index fa032a3a2e..ac71be37ba 100644 --- a/internal/core/src/storage/Util.cpp +++ b/internal/core/src/storage/Util.cpp @@ -44,7 +44,7 @@ #include "storage/InsertData.h" #include "storage/LocalChunkManager.h" #include "storage/MemFileManagerImpl.h" -#include "storage/MinioChunkManager.h" +#include "storage/minio/MinioChunkManager.h" #ifdef USE_OPENDAL #include "storage/opendal/OpenDALChunkManager.h" #endif diff --git a/internal/core/src/storage/AliyunCredentialsProvider.cpp b/internal/core/src/storage/aliyun/AliyunCredentialsProvider.cpp similarity index 100% rename from internal/core/src/storage/AliyunCredentialsProvider.cpp rename to internal/core/src/storage/aliyun/AliyunCredentialsProvider.cpp diff --git a/internal/core/src/storage/AliyunCredentialsProvider.h b/internal/core/src/storage/aliyun/AliyunCredentialsProvider.h similarity index 100% rename from internal/core/src/storage/AliyunCredentialsProvider.h rename to internal/core/src/storage/aliyun/AliyunCredentialsProvider.h diff --git a/internal/core/src/storage/AliyunSTSClient.cpp b/internal/core/src/storage/aliyun/AliyunSTSClient.cpp similarity index 100% rename from internal/core/src/storage/AliyunSTSClient.cpp rename to internal/core/src/storage/aliyun/AliyunSTSClient.cpp diff --git a/internal/core/src/storage/AliyunSTSClient.h b/internal/core/src/storage/aliyun/AliyunSTSClient.h similarity index 100% rename from internal/core/src/storage/AliyunSTSClient.h rename to internal/core/src/storage/aliyun/AliyunSTSClient.h diff --git a/internal/core/src/storage/aliyun/CMakeLists.txt b/internal/core/src/storage/aliyun/CMakeLists.txt new file mode 100644 index 0000000000..c22b70e9b1 --- /dev/null +++ b/internal/core/src/storage/aliyun/CMakeLists.txt @@ -0,0 +1,19 @@ +# Copyright (C) 2019-2020 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under the License + +set(ALIYUN_SRCS + ${CMAKE_CURRENT_SOURCE_DIR}/AliyunCredentialsProvider.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/AliyunSTSClient.cpp +) + +set(SOURCE_FILES ${SOURCE_FILES} ${ALIYUN_SRCS} PARENT_SCOPE) + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/internal/core/src/storage/azure/AzureBlobChunkManager.cpp b/internal/core/src/storage/azure/AzureBlobChunkManager.cpp index 9db84eedc7..94d0ca52bd 100644 --- a/internal/core/src/storage/azure/AzureBlobChunkManager.cpp +++ b/internal/core/src/storage/azure/AzureBlobChunkManager.cpp @@ -14,8 +14,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include +#include #include #include +#include #include #include "AzureBlobChunkManager.h" @@ -66,30 +69,51 @@ AzureBlobChunkManager::AzureBlobChunkManager( bool useIAM) { requestTimeoutMs_ = requestTimeoutMs; if (useIAM) { - Azure::Identity::WorkloadIdentityCredentialOptions options; - auto workloadIdentityCredential = - std::make_shared( - options); - client_ = std::make_shared( - "https://" + access_key_id + ".blob." + address + "/", - workloadIdentityCredential); + const char* federated_token_file = + std::getenv("AZURE_FEDERATED_TOKEN_FILE"); + const bool has_federated_token = federated_token_file != nullptr && + std::strlen(federated_token_file) > 0; + if (has_federated_token) { + Azure::Identity::WorkloadIdentityCredentialOptions options; + const char* workload_client_id = std::getenv("AZURE_CLIENT_ID"); + const char* workload_tenant_id = std::getenv("AZURE_TENANT_ID"); + options.ClientId = + workload_client_id == nullptr ? "" : workload_client_id; + options.TenantId = + workload_tenant_id == nullptr ? "" : workload_tenant_id; + options.TokenFilePath = federated_token_file; + auto workloadIdentityCredential = + std::make_shared( + options); + client_ = + std::make_shared( + "https://" + access_key_id + ".blob." + address + "/", + workloadIdentityCredential); + } else { + const char* client_id_env = std::getenv("AZURE_CLIENT_ID"); + std::string client_id = + client_id_env == nullptr ? "" : client_id_env; + std::shared_ptr + credential; + if (client_id.empty()) { + credential = std::make_shared< + Azure::Identity::ManagedIdentityCredential>(); + } else { + credential = std::make_shared< + Azure::Identity::ManagedIdentityCredential>(client_id); + } + + client_ = + std::make_shared( + "https://" + access_key_id + ".blob." + address + "/", + credential); + } } else { client_ = std::make_shared( Azure::Storage::Blobs::BlobServiceClient:: CreateFromConnectionString(GetConnectionString( access_key_id, access_key_value, address))); } - try { - Azure::Core::Context context; - client_->GetBlobContainerClient("justforconnectioncheck") - .GetBlockBlobClient("justforconnectioncheck") - .GetProperties(Azure::Storage::Blobs::GetBlobPropertiesOptions(), - context); - } catch (const Azure::Storage::StorageException& e) { - if (e.StatusCode != Azure::Core::Http::HttpStatusCode::NotFound) { - throw; - } - } } AzureBlobChunkManager::~AzureBlobChunkManager() { diff --git a/internal/core/src/storage/huawei/CMakeLists.txt b/internal/core/src/storage/huawei/CMakeLists.txt new file mode 100644 index 0000000000..1930a17107 --- /dev/null +++ b/internal/core/src/storage/huawei/CMakeLists.txt @@ -0,0 +1,19 @@ +# Copyright (C) 2019-2020 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under the License + +set(HUAWEI_SRCS + ${CMAKE_CURRENT_SOURCE_DIR}/HuaweiCloudCredentialsProvider.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/HuaweiCloudSTSClient.cpp +) + +set(SOURCE_FILES ${SOURCE_FILES} ${HUAWEI_SRCS} PARENT_SCOPE) + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/internal/core/src/storage/HuaweiCloudCredentialsProvider.cpp b/internal/core/src/storage/huawei/HuaweiCloudCredentialsProvider.cpp similarity index 100% rename from internal/core/src/storage/HuaweiCloudCredentialsProvider.cpp rename to internal/core/src/storage/huawei/HuaweiCloudCredentialsProvider.cpp diff --git a/internal/core/src/storage/HuaweiCloudCredentialsProvider.h b/internal/core/src/storage/huawei/HuaweiCloudCredentialsProvider.h similarity index 100% rename from internal/core/src/storage/HuaweiCloudCredentialsProvider.h rename to internal/core/src/storage/huawei/HuaweiCloudCredentialsProvider.h diff --git a/internal/core/src/storage/HuaweiCloudSTSClient.cpp b/internal/core/src/storage/huawei/HuaweiCloudSTSClient.cpp similarity index 100% rename from internal/core/src/storage/HuaweiCloudSTSClient.cpp rename to internal/core/src/storage/huawei/HuaweiCloudSTSClient.cpp diff --git a/internal/core/src/storage/HuaweiCloudSTSClient.h b/internal/core/src/storage/huawei/HuaweiCloudSTSClient.h similarity index 100% rename from internal/core/src/storage/HuaweiCloudSTSClient.h rename to internal/core/src/storage/huawei/HuaweiCloudSTSClient.h diff --git a/internal/core/src/storage/minio/CMakeLists.txt b/internal/core/src/storage/minio/CMakeLists.txt new file mode 100644 index 0000000000..9111ad1ecf --- /dev/null +++ b/internal/core/src/storage/minio/CMakeLists.txt @@ -0,0 +1,18 @@ +# Copyright (C) 2019-2020 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under the License + +set(MINIO_SRCS + ${CMAKE_CURRENT_SOURCE_DIR}/MinioChunkManager.cpp +) + +set(SOURCE_FILES ${SOURCE_FILES} ${MINIO_SRCS} PARENT_SCOPE) + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/internal/core/src/storage/MinioChunkManager.cpp b/internal/core/src/storage/minio/MinioChunkManager.cpp similarity index 99% rename from internal/core/src/storage/MinioChunkManager.cpp rename to internal/core/src/storage/minio/MinioChunkManager.cpp index c6cf20f45f..c55e3b0e00 100644 --- a/internal/core/src/storage/MinioChunkManager.cpp +++ b/internal/core/src/storage/minio/MinioChunkManager.cpp @@ -14,7 +14,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "storage/MinioChunkManager.h" +#include "storage/minio/MinioChunkManager.h" #include #include @@ -30,10 +30,10 @@ #include #include -#include "storage/AliyunSTSClient.h" -#include "storage/AliyunCredentialsProvider.h" -#include "storage/TencentCloudSTSClient.h" -#include "storage/TencentCloudCredentialsProvider.h" +#include "storage/aliyun/AliyunSTSClient.h" +#include "storage/aliyun/AliyunCredentialsProvider.h" +#include "storage/tencent/TencentCloudSTSClient.h" +#include "storage/tencent/TencentCloudCredentialsProvider.h" #include "monitor/Monitor.h" #include "common/EasyAssert.h" #include "log/Log.h" diff --git a/internal/core/src/storage/MinioChunkManager.h b/internal/core/src/storage/minio/MinioChunkManager.h similarity index 100% rename from internal/core/src/storage/MinioChunkManager.h rename to internal/core/src/storage/minio/MinioChunkManager.h diff --git a/internal/core/src/storage/MinioChunkManagerTest.cpp b/internal/core/src/storage/minio/MinioChunkManagerTest.cpp similarity index 99% rename from internal/core/src/storage/MinioChunkManagerTest.cpp rename to internal/core/src/storage/minio/MinioChunkManagerTest.cpp index 9361ff4f02..c3ab1c211b 100644 --- a/internal/core/src/storage/MinioChunkManagerTest.cpp +++ b/internal/core/src/storage/minio/MinioChunkManagerTest.cpp @@ -13,7 +13,7 @@ #include #include -#include "storage/MinioChunkManager.h" +#include "storage/minio/MinioChunkManager.h" #include "test_utils/indexbuilder_test_utils.h" using namespace std; diff --git a/internal/core/src/storage/tencent/CMakeLists.txt b/internal/core/src/storage/tencent/CMakeLists.txt new file mode 100644 index 0000000000..14a3e25a11 --- /dev/null +++ b/internal/core/src/storage/tencent/CMakeLists.txt @@ -0,0 +1,19 @@ +# Copyright (C) 2019-2020 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under the License + +set(TENCENT_SRCS + ${CMAKE_CURRENT_SOURCE_DIR}/TencentCloudCredentialsProvider.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/TencentCloudSTSClient.cpp +) + +set(SOURCE_FILES ${SOURCE_FILES} ${TENCENT_SRCS} PARENT_SCOPE) + +include_directories(${CMAKE_CURRENT_SOURCE_DIR}) diff --git a/internal/core/src/storage/TencentCloudCredentialsProvider.cpp b/internal/core/src/storage/tencent/TencentCloudCredentialsProvider.cpp similarity index 100% rename from internal/core/src/storage/TencentCloudCredentialsProvider.cpp rename to internal/core/src/storage/tencent/TencentCloudCredentialsProvider.cpp diff --git a/internal/core/src/storage/TencentCloudCredentialsProvider.h b/internal/core/src/storage/tencent/TencentCloudCredentialsProvider.h similarity index 100% rename from internal/core/src/storage/TencentCloudCredentialsProvider.h rename to internal/core/src/storage/tencent/TencentCloudCredentialsProvider.h diff --git a/internal/core/src/storage/TencentCloudSTSClient.cpp b/internal/core/src/storage/tencent/TencentCloudSTSClient.cpp similarity index 100% rename from internal/core/src/storage/TencentCloudSTSClient.cpp rename to internal/core/src/storage/tencent/TencentCloudSTSClient.cpp diff --git a/internal/core/src/storage/TencentCloudSTSClient.h b/internal/core/src/storage/tencent/TencentCloudSTSClient.h similarity index 100% rename from internal/core/src/storage/TencentCloudSTSClient.h rename to internal/core/src/storage/tencent/TencentCloudSTSClient.h diff --git a/internal/proxy/task_index.go b/internal/proxy/task_index.go index 19c85e8327..ee29011793 100644 --- a/internal/proxy/task_index.go +++ b/internal/proxy/task_index.go @@ -211,7 +211,7 @@ func (cit *createIndexTask) parseIndexParams(ctx context.Context) error { checker, err := indexparamcheck.GetIndexCheckerMgrInstance().GetChecker(specifyIndexType) // not enable hybrid index for user, used in milvus internally if err != nil || indexparamcheck.IsHYBRIDChecker(checker) { - log.Ctx(ctx).Warn("Failed to get index checker", zap.String(common.IndexTypeKey, specifyIndexType)) + log.Ctx(ctx).Warn("Failed to get index checker", zap.String(common.IndexTypeKey, specifyIndexType), zap.Error(err)) return merr.WrapErrParameterInvalid("valid index", fmt.Sprintf("invalid index type: %s", specifyIndexType)) } } diff --git a/pkg/objectstorage/util.go b/pkg/objectstorage/util.go index 7dbfd7645f..b2f9047153 100644 --- a/pkg/objectstorage/util.go +++ b/pkg/objectstorage/util.go @@ -168,11 +168,26 @@ func NewAzureObjectStorageClient(ctx context.Context, c *Config) (*service.Clien var client *service.Client var err error if c.UseIAM { - cred, credErr := azidentity.NewWorkloadIdentityCredential(&azidentity.WorkloadIdentityCredentialOptions{ - ClientID: os.Getenv("AZURE_CLIENT_ID"), - TenantID: os.Getenv("AZURE_TENANT_ID"), - TokenFilePath: os.Getenv("AZURE_FEDERATED_TOKEN_FILE"), - }) + var cred azcore.TokenCredential + var credErr error + if os.Getenv("AZURE_FEDERATED_TOKEN_FILE") != "" { + cred, credErr = azidentity.NewWorkloadIdentityCredential(&azidentity.WorkloadIdentityCredentialOptions{ + ClientID: os.Getenv("AZURE_CLIENT_ID"), + TenantID: os.Getenv("AZURE_TENANT_ID"), + TokenFilePath: os.Getenv("AZURE_FEDERATED_TOKEN_FILE"), + }) + } else { + clientID := os.Getenv("AZURE_CLIENT_ID") + managedIdentityID := azidentity.ClientID("") // Default to System Assigned + + if clientID != "" { + managedIdentityID = azidentity.ClientID(clientID) + } + + cred, credErr = azidentity.NewManagedIdentityCredential(&azidentity.ManagedIdentityCredentialOptions{ + ID: managedIdentityID, + }) + } if credErr != nil { return nil, credErr }