diff --git a/internal/core/src/CMakeLists.txt b/internal/core/src/CMakeLists.txt index a56b32dccf..28705e55ff 100644 --- a/internal/core/src/CMakeLists.txt +++ b/internal/core/src/CMakeLists.txt @@ -84,7 +84,8 @@ if(USE_OPENDAL) set(LINK_TARGETS ${LINK_TARGETS} opendal) endif() -if(DEFINED AZURE_BUILD_DIR) +if(ENABLE_AZURE_FS) + set(AZURE_BUILD_DIR ON) # Arrow already includes Azure SDK when built with_azure=True # No need to link additional azure_blob_chunk_manager endif() diff --git a/internal/core/src/storage/CMakeLists.txt b/internal/core/src/storage/CMakeLists.txt index 350d3d831d..614a837658 100644 --- a/internal/core/src/storage/CMakeLists.txt +++ b/internal/core/src/storage/CMakeLists.txt @@ -21,7 +21,8 @@ if (ENABLE_GCP_NATIVE) add_subdirectory(gcp-native-storage) endif() -if (DEFINED AZURE_BUILD_DIR) +if (ENABLE_AZURE_FS) + set(AZURE_BUILD_DIR ON) add_definitions(-DAZURE_BUILD_DIR) add_subdirectory(azure) endif() diff --git a/internal/core/src/storage/azure/AzureBlobChunkManagerTest.cpp b/internal/core/src/storage/azure/AzureBlobChunkManagerTest.cpp index be1113f8b1..f7da8bf5b8 100644 --- a/internal/core/src/storage/azure/AzureBlobChunkManagerTest.cpp +++ b/internal/core/src/storage/azure/AzureBlobChunkManagerTest.cpp @@ -65,67 +65,3 @@ TEST(AzureBlobChunkManagerTest, Options) { EXPECT_EQ(options.AuthorityHost, "https://login.microsoftonline.com/"); EXPECT_EQ(options.TokenFilePath, GetTokenFilePath()); } - -void -print(Azure::Core::Diagnostics::Logger::Level level, - std::string const& message) { - std::cout << "level: " - << ", message: " << message << std::endl; -} - -int -main0() { - const char* containerName = "default"; - const char* blobName = "sample-blob"; - AzureBlobChunkManager::InitLog("info", print); - AzureBlobChunkManager chunkManager = AzureBlobChunkManager("", "", ""); - std::vector buckets = chunkManager.ListBuckets(); - std::cout << "list buckets." << std::endl; - for (const auto& bucket : buckets) { - std::cout << bucket << std::endl; - } - if (!chunkManager.BucketExists(containerName)) { - std::cout << "create a bucket named: " << containerName << std::endl; - chunkManager.CreateBucket(containerName); - } else { - std::vector objects = - chunkManager.ListObjects(containerName, blobName); - for (const auto& object : objects) { - std::cout << object << std::endl; - } - } - std::cout << chunkManager.BucketExists(containerName) << std::endl; - - if (!chunkManager.ObjectExists(containerName, blobName)) { - char msg[12]; - memcpy(msg, "Azure hello!", 12); - chunkManager.PutObjectBuffer(containerName, blobName, msg, 12); - } - std::cout << chunkManager.GetObjectSize(containerName, blobName) - << std::endl; - std::cout << chunkManager.ObjectExists(containerName, blobName) - << std::endl; - std::cout << chunkManager.ObjectExists(containerName, "blobName") - << std::endl; - char buffer[1024 * 1024]; - chunkManager.GetObjectBuffer(containerName, blobName, buffer, 1024 * 1024); - std::cout << buffer << std::endl; - chunkManager.DeleteObject(containerName, blobName); - try { - chunkManager.GetObjectBuffer( - containerName, "blobName", buffer, 1024 * 1024); - } catch (const std::exception& e) { - std::cout << "object('" << containerName << "', 'blobName') not exists" - << e.what() << std::endl; - } - std::cout << "create a bucket duplicated " - << chunkManager.CreateBucket(containerName) << std::endl; - chunkManager.DeleteBucket(containerName); - exit(EXIT_SUCCESS); -} - -int -main(int argc, char** argv) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} diff --git a/internal/core/src/storage/azure/AzureChunkManagerTest.cpp b/internal/core/src/storage/azure/AzureChunkManagerTest.cpp index b02770ef12..cfd8c777e7 100644 --- a/internal/core/src/storage/azure/AzureChunkManagerTest.cpp +++ b/internal/core/src/storage/azure/AzureChunkManagerTest.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include "common/EasyAssert.h" #include "AzureChunkManager.h" @@ -21,9 +22,22 @@ using namespace std; using namespace milvus; using namespace milvus::storage; +// the accessKey and accessValue is the the fixed storage account used by Azure Storage Emulator StorageConfig get_default_storage_config(bool useIam) { - auto endpoint = "core.windows.net"; + // Set AZURE_STORAGE_CONNECTION_STRING for local testing with Azurite + if (!std::getenv("AZURE_STORAGE_CONNECTION_STRING")) { + setenv("AZURE_STORAGE_CONNECTION_STRING", + "DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;" + "AccountKey=" + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/" + "K1SZFPTOtr/KBHBeksoGMGw==;" + "BlobEndpoint=http://127.0.0.1:10000/devstoreaccount1;", + 1); + } + + auto endpoint = + "127.0.0.1:10000"; // Not actually used in azure chunk managerwhen connection string is set auto accessKey = "devstoreaccount1"; auto accessValue = "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/" @@ -69,18 +83,6 @@ class AzureChunkManagerTest : public testing::Test { StorageConfig configs_; }; -TEST_F(AzureChunkManagerTest, WrongConfig) { - StorageConfig configs = get_default_storage_config(true); - - try { - AzureChunkManagerPtr chunk_manager = - make_unique(configs); - EXPECT_TRUE(false); - } catch (SegcoreError& e) { - EXPECT_TRUE(std::string(e.what()).find("precheck") != string::npos); - } -} - TEST_F(AzureChunkManagerTest, AzureLogger) { AzureLogger(Azure::Core::Diagnostics::Logger::Level::Error, ""); AzureLogger(Azure::Core::Diagnostics::Logger::Level::Warning, ""); diff --git a/internal/core/unittest/CMakeLists.txt b/internal/core/unittest/CMakeLists.txt index 08634448e1..44d866bd4d 100644 --- a/internal/core/unittest/CMakeLists.txt +++ b/internal/core/unittest/CMakeLists.txt @@ -74,7 +74,8 @@ if (NOT (LINUX OR APPLE)) list(FILTER MILVUS_TEST_FILES EXCLUDE REGEX "ExprArrayTest\\.cpp$") endif() -if (DEFINED AZURE_BUILD_DIR) +if (ENABLE_AZURE_FS) + set(AZURE_BUILD_DIR ON) add_definitions(-DAZURE_BUILD_DIR) else() list(FILTER MILVUS_TEST_FILES EXCLUDE REGEX "AzureChunkManagerTest\\.cpp$")