admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: turn on azure by default (#44377)

Browse Source 
related: #44354, #44138, #43869

---------

Signed-off-by: shaoting-huang <shaoting.huang@zilliz.com>
This commit is contained in:
sthuang 2025-09-17 10:12:01 +08:00 committed by GitHub
parent 6f7318a731
commit 2f70a73258
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 21 additions and 80 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
internal/core/src/CMakeLists.txt
View File

@ -84,7 +84,8 @@ if(USE_OPENDAL)
set(LINK_TARGETS ${LINK_TARGETS} opendal)
endif()
if(DEFINED AZURE_BUILD_DIR)
if(ENABLE_AZURE_FS)
set(AZURE_BUILD_DIR ON)
# Arrow already includes Azure SDK when built with_azure=True
# No need to link additional azure_blob_chunk_manager
endif()

3
internal/core/src/storage/CMakeLists.txt
View File

@ -21,7 +21,8 @@ if (ENABLE_GCP_NATIVE)
add_subdirectory(gcp-native-storage)
endif()
if (DEFINED AZURE_BUILD_DIR)
if (ENABLE_AZURE_FS)
set(AZURE_BUILD_DIR ON)
add_definitions(-DAZURE_BUILD_DIR)
add_subdirectory(azure)
endif()

64
internal/core/src/storage/azure/AzureBlobChunkManagerTest.cpp
View File

@ -65,67 +65,3 @@ TEST(AzureBlobChunkManagerTest, Options) {
EXPECT_EQ(options.AuthorityHost, "https://login.microsoftonline.com/");
EXPECT_EQ(options.TokenFilePath, GetTokenFilePath());
}
void
print(Azure::Core::Diagnostics::Logger::Level level,
std::string const& message) {
std::cout << "level: "
<< ", message: " << message << std::endl;
}
int
main0() {
const char* containerName = "default";
const char* blobName = "sample-blob";
AzureBlobChunkManager::InitLog("info", print);
AzureBlobChunkManager chunkManager = AzureBlobChunkManager("", "", "");
std::vector<std::string> buckets = chunkManager.ListBuckets();
std::cout << "list buckets." << std::endl;
for (const auto& bucket : buckets) {
std::cout << bucket << std::endl;
}
if (!chunkManager.BucketExists(containerName)) {
std::cout << "create a bucket named: " << containerName << std::endl;
chunkManager.CreateBucket(containerName);
} else {
std::vector<std::string> objects =
chunkManager.ListObjects(containerName, blobName);
for (const auto& object : objects) {
std::cout << object << std::endl;
}
}
std::cout << chunkManager.BucketExists(containerName) << std::endl;
if (!chunkManager.ObjectExists(containerName, blobName)) {
char msg[12];
memcpy(msg, "Azure hello!", 12);
chunkManager.PutObjectBuffer(containerName, blobName, msg, 12);
}
std::cout << chunkManager.GetObjectSize(containerName, blobName)
<< std::endl;
std::cout << chunkManager.ObjectExists(containerName, blobName)
<< std::endl;
std::cout << chunkManager.ObjectExists(containerName, "blobName")
<< std::endl;
char buffer[1024 * 1024];
chunkManager.GetObjectBuffer(containerName, blobName, buffer, 1024 * 1024);
std::cout << buffer << std::endl;
chunkManager.DeleteObject(containerName, blobName);
try {
chunkManager.GetObjectBuffer(
containerName, "blobName", buffer, 1024 * 1024);
} catch (const std::exception& e) {
std::cout << "object('" << containerName << "', 'blobName') not exists"
<< e.what() << std::endl;
}
std::cout << "create a bucket duplicated "
<< chunkManager.CreateBucket(containerName) << std::endl;
chunkManager.DeleteBucket(containerName);
exit(EXIT_SUCCESS);
}
int
main(int argc, char** argv) {
testing::InitGoogleTest(&argc, argv);
return RUN_ALL_TESTS();
}

28
internal/core/src/storage/azure/AzureChunkManagerTest.cpp
View File

@ -12,6 +12,7 @@
#include <gtest/gtest.h>
#include <string>
#include <vector>
#include <cstdlib>
#include "common/EasyAssert.h"
#include "AzureChunkManager.h"
@ -21,9 +22,22 @@ using namespace std;
using namespace milvus;
using namespace milvus::storage;
// the accessKey and accessValue is the the fixed storage account used by Azure Storage Emulator
StorageConfig
get_default_storage_config(bool useIam) {
auto endpoint = "core.windows.net";
// Set AZURE_STORAGE_CONNECTION_STRING for local testing with Azurite
if (!std::getenv("AZURE_STORAGE_CONNECTION_STRING")) {
setenv("AZURE_STORAGE_CONNECTION_STRING",
"DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;"
"AccountKey="
"Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/"
"K1SZFPTOtr/KBHBeksoGMGw==;"
"BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;",
1);
}
auto endpoint =
"127.0.0.1:10000"; // Not actually used in azure chunk managerwhen connection string is set
auto accessKey = "devstoreaccount1";
auto accessValue =
"Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/"
@ -69,18 +83,6 @@ class AzureChunkManagerTest : public testing::Test {
StorageConfig configs_;
};
TEST_F(AzureChunkManagerTest, WrongConfig) {
StorageConfig configs = get_default_storage_config(true);
try {
AzureChunkManagerPtr chunk_manager =
make_unique<AzureChunkManager>(configs);
EXPECT_TRUE(false);
} catch (SegcoreError& e) {
EXPECT_TRUE(std::string(e.what()).find("precheck") != string::npos);
}
}
TEST_F(AzureChunkManagerTest, AzureLogger) {
AzureLogger(Azure::Core::Diagnostics::Logger::Level::Error, "");
AzureLogger(Azure::Core::Diagnostics::Logger::Level::Warning, "");

3
internal/core/unittest/CMakeLists.txt
View File

@ -74,7 +74,8 @@ if (NOT (LINUX OR APPLE))
list(FILTER MILVUS_TEST_FILES EXCLUDE REGEX "ExprArrayTest\\.cpp$")
endif()
if (DEFINED AZURE_BUILD_DIR)
if (ENABLE_AZURE_FS)
set(AZURE_BUILD_DIR ON)
add_definitions(-DAZURE_BUILD_DIR)
else()
list(FILTER MILVUS_TEST_FILES EXCLUDE REGEX "AzureChunkManagerTest\\.cpp$")