enhance: use already installed vcpkg (#28703)

issue #28686
1. Update Builder gpu image changes, see changes #28505
2. update azure-identity-cpp from beta to release

Signed-off-by: PowderLi <min.li@zilliz.com>

.env

@@ -1,10 +1,10 @@
 IMAGE_REPO=milvusdb
 IMAGE_ARCH=amd64
 OS_NAME=ubuntu20.04
-DATE_VERSION=20231024-4faba61
+DATE_VERSION=20231123-90997a5
-LATEST_DATE_VERSION=20231024-4faba61
+LATEST_DATE_VERSION=20231123-90997a5
-GPU_DATE_VERSION=20231123-90997a5
+GPU_DATE_VERSION=20231025-a951c75
-LATEST_GPU_DATE_VERSION=20231123-90997a5
+LATEST_GPU_DATE_VERSION=20231025-a951c75
 MINIO_ADDRESS=minio:9000
 PULSAR_ADDRESS=pulsar://pulsar:6650
 ETCD_ENDPOINTS=etcd:2379

.github/workflows/publish-builder.yaml

@@ -7,14 +7,12 @@ on:
     # file paths to consider in the event. Optional; defaults to all.
     paths:
       - 'build/docker/builder/cpu/**'
-      - 'build/docker/builder/gpu/**'
       - '.github/workflows/publish-builder.yaml'
       - '!**.md'
   pull_request:
     # file paths to consider in the event. Optional; defaults to all.
     paths:
       - 'build/docker/builder/cpu/**'
-      - 'build/docker/builder/gpu/**'
       - '.github/workflows/publish-builder.yaml'
       - '!**.md'
 
@@ -55,7 +53,6 @@ jobs:
           docker buildx build --platform linux/amd64,linux/arm64 -t milvusdb/milvus-env:${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/cpu/${OS_NAME}/Dockerfile .      
       - name: Docker Build&Push
         if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
-        continue-on-error: true
         shell: bash
         run: |
           docker buildx ls
@@ -64,7 +61,7 @@ jobs:
           docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/cpu/${OS_NAME}/Dockerfile .
           docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:${OS_NAME}-latest -f build/docker/builder/cpu/${OS_NAME}/Dockerfile .
       - name: Update Builder Image Changes
-        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
+        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04'
         continue-on-error: true
         shell: bash
         run: |
@@ -96,80 +93,3 @@ jobs:
           echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
           echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"
 
-
-  publish-gpu-builder:
-    name: ${{ matrix.arch }} ${{ matrix.os }}
-    runs-on: ubuntu-latest
-    timeout-minutes: 500
-    strategy:
-      fail-fast: false
-      matrix:
-        os: [ ubuntu20.04 ]
-        arch: [ amd64&arm64 ]
-    env:
-      OS_NAME: ${{ matrix.os }}
-      IMAGE_ARCH: ${{ matrix.arch }}
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-      - name: Get version from system time after release step
-        id: extracter
-        run: |
-          echo "::set-output name=version::$(date +%Y%m%d)"
-          echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
-      # - name: Setup upterm session
-      #   uses: lhotari/action-upterm@v1
-      - name: Docker Pull
-        shell: bash
-        run: |
-          docker run --rm --privileged tonistiigi/binfmt:latest --install arm64
-          docker buildx ls
-          docker buildx create --use --name=milvus --driver docker-container
-      - name: Docker Build
-        if: success() && github.event_name == 'pull_request' && github.repository == 'milvus-io/milvus'
-        shell: bash
-        run: |
-          docker buildx ls
-          docker buildx build --platform linux/amd64,linux/arm64 -t milvusdb/milvus-env:gpu-${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/gpu/${OS_NAME}/Dockerfile .
-      - name: Docker Build&Push
-        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
-        continue-on-error: true
-        shell: bash
-        run: |
-          docker buildx ls
-          docker login -u ${{ secrets.DOCKERHUB_USER }} \
-                       -p ${{ secrets.DOCKERHUB_TOKEN }}
-          docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:gpu-${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/gpu/${OS_NAME}/Dockerfile .
-          docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:gpu-${OS_NAME}-latest -f build/docker/builder/gpu/${OS_NAME}/Dockerfile .
-      - name: Update Builder Image Changes
-        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
-        continue-on-error: true
-        shell: bash
-        run: |
-          sed -i "s#^GPU_DATE_VERSION=.*#GPU_DATE_VERSION=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" .env
-          sed -i "s#^LATEST_GPU_DATE_VERSION=.*#LATEST_GPU_DATE_VERSION=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" .env
-          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git config --local user.name "github-actions[bot]"
-          git add .env
-          git commit -m "Update Builder gpu image changes"
-      - name: Create Pull Request
-        id: cpr
-        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04'
-        continue-on-error: true
-        uses: peter-evans/create-pull-request@v3
-        with:
-          token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }}
-          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
-          signoff: false
-          branch: update_builder_${{ github.sha }}
-          delete-branch: true
-          title: '[automated] Update Builder gpu image changes'
-          body: |
-            Update Builder gpu image changes
-            See changes: https://github.com/milvus-io/milvus/commit/${{ github.sha }}
-            Signed-off-by: ${{ github.actor }} ${{ github.actor }}@users.noreply.github.com
-      - name: Check outputs
-        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04'
-        run: |
-          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
-          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"

.github/workflows/publish-gpu-builder.yaml

@@ -0,0 +1,94 @@
+name: Publish Gpu Builder
+# TODO: do not trigger action for some document file update
+
+# This workflow is triggered on pushes or pull request to the repository.
+on:
+  push:
+    # file paths to consider in the event. Optional; defaults to all.
+    paths:
+      - 'build/docker/builder/gpu/**'
+      - '.github/workflows/publish-gpu-builder.yaml'
+      - '!**.md'
+  pull_request:
+    # file paths to consider in the event. Optional; defaults to all.
+    paths:
+      - 'build/docker/builder/gpu/**'
+      - '.github/workflows/publish-gpu-builder.yaml'
+      - '!**.md'
+
+jobs:
+  publish-gpu-builder:
+    name: ${{ matrix.arch }} ${{ matrix.os }}
+    runs-on: ubuntu-latest
+    timeout-minutes: 500
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu20.04]
+        arch: [amd64&arm64]
+    env:
+      OS_NAME: ${{ matrix.os }}
+      IMAGE_ARCH: ${{ matrix.arch }}
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Get version from system time after release step
+        id: extracter
+        run: |
+          echo "::set-output name=version::$(date +%Y%m%d)"
+          echo "::set-output name=sha_short::$(git rev-parse --short HEAD)"
+      # - name: Setup upterm session
+      #   uses: lhotari/action-upterm@v1
+      - name: Docker Pull
+        shell: bash
+        run: |
+          docker run --rm --privileged tonistiigi/binfmt:latest --install arm64
+          docker buildx ls
+          docker buildx create --use --name=milvus --driver docker-container
+      - name: Docker Build
+        if: success() && github.event_name == 'pull_request' && github.repository == 'milvus-io/milvus'
+        shell: bash
+        run: |
+          docker buildx ls
+          docker buildx build --platform linux/amd64,linux/arm64 -t milvusdb/milvus-env:gpu-${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/gpu/${OS_NAME}/Dockerfile .
+      - name: Docker Build&Push
+        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus'
+        shell: bash
+        run: |
+          docker buildx ls
+          docker login -u ${{ secrets.DOCKERHUB_USER }} \
+                       -p ${{ secrets.DOCKERHUB_TOKEN }}
+          docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:gpu-${OS_NAME}-${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }} -f build/docker/builder/gpu/${OS_NAME}/Dockerfile .
+          docker buildx build --platform linux/amd64,linux/arm64 --push -t milvusdb/milvus-env:gpu-${OS_NAME}-latest -f build/docker/builder/gpu/${OS_NAME}/Dockerfile .
+      - name: Update Builder Image Changes
+        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04'
+        continue-on-error: true
+        shell: bash
+        run: |
+          sed -i "s#^GPU_DATE_VERSION=.*#GPU_DATE_VERSION=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" .env
+          sed -i "s#^LATEST_GPU_DATE_VERSION=.*#LATEST_GPU_DATE_VERSION=${{ steps.extracter.outputs.version }}-${{ steps.extracter.outputs.sha_short }}#g" .env
+          git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com"
+          git config --local user.name "github-actions[bot]"
+          git add .env
+          git commit -m "Update Builder gpu image changes"
+      - name: Create Pull Request
+        id: cpr
+        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04'
+        continue-on-error: true
+        uses: peter-evans/create-pull-request@v3
+        with:
+          token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }}
+          author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com>
+          signoff: false
+          branch: update_gpu_builder_${{ github.sha }}
+          delete-branch: true
+          title: '[automated] Update Builder gpu image changes'
+          body: |
+            Update Builder gpu image changes
+            See changes: https://github.com/milvus-io/milvus/commit/${{ github.sha }}
+            Signed-off-by: ${{ github.actor }} ${{ github.actor }}@users.noreply.github.com
+      - name: Check outputs
+        if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' && matrix.os == 'ubuntu20.04'
+        run: |
+          echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}"
+          echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}"

docker-compose.yml

@@ -14,6 +14,8 @@ services:
     build:
       context: .
       dockerfile: build/docker/builder/cpu/${OS_NAME}/Dockerfile
+      args:
+        TARGETARCH: ${IMAGE_ARCH}
       cache_from:
         - ${IMAGE_REPO}/milvus-env:${OS_NAME}-${LATEST_DATE_VERSION}
     platform: linux/${IMAGE_ARCH}
@@ -52,6 +54,8 @@ services:
     build:
       context: .
       dockerfile: build/docker/builder/gpu/${OS_NAME}/Dockerfile
+      args:
+        TARGETARCH: ${IMAGE_ARCH}
       cache_from:
         - ${IMAGE_REPO}/milvus-env:gpu-${OS_NAME}-${LATEST_GPU_DATE_VERSION}
     # user: {{ CURRENT_ID }}

internal/core/src/storage/azure-blob-storage/AzureBlobChunkManager.cpp

@@ -21,18 +21,6 @@
 
 namespace azure {
 
-std::string
-GetTenantId() {
-    return std::getenv("AZURE_TENANT_ID");
-}
-std::string
-GetClientId() {
-    return std::getenv("AZURE_CLIENT_ID");
-}
-std::string
-GetTokenFilePath() {
-    return std::getenv("AZURE_FEDERATED_TOKEN_FILE");
-}
 std::string
 GetConnectionString(const std::string& access_key_id,
                     const std::string& access_key_value,
@@ -78,9 +66,10 @@ AzureBlobChunkManager::AzureBlobChunkManager(
     bool useIAM) {
     requestTimeoutMs_ = requestTimeoutMs;
     if (useIAM) {
+        Azure::Identity::WorkloadIdentityCredentialOptions options;
         auto workloadIdentityCredential =
             std::make_shared<Azure::Identity::WorkloadIdentityCredential>(
-                GetTenantId(), GetClientId(), GetTokenFilePath());
+                options);
         client_ = std::make_shared<Azure::Storage::Blobs::BlobServiceClient>(
             "https://" + access_key_id + ".blob." + address + "/",
             workloadIdentityCredential);

internal/core/src/storage/azure-blob-storage/cmake-modules/AzureVcpkg.cmake

@@ -6,6 +6,7 @@ set(AZ_ROOT_DIR "${CMAKE_CURRENT_LIST_DIR}/..")
 
 macro(az_vcpkg_integrate)
   message("Vcpkg integrate step.")
+
   # AUTO CMAKE_TOOLCHAIN_FILE:
   #   User can call `cmake -DCMAKE_TOOLCHAIN_FILE="path_to_the_toolchain"` as the most specific scenario.
   #   As the last alternative (default case), Azure SDK will automatically clone VCPKG folder and set toolchain from there.
@@ -17,7 +18,7 @@ macro(az_vcpkg_integrate)
       message("AZURE_SDK_DISABLE_AUTO_VCPKG is not defined. Fetch a local copy of vcpkg.")
       # GET VCPKG FROM SOURCE
       #  User can set env var AZURE_SDK_VCPKG_COMMIT to pick the VCPKG commit to fetch
-      set(VCPKG_COMMIT_STRING 71d875654e32ee216b0b7e0dc684e589dffa1b1c) # default SDK tested commit
+      set(VCPKG_COMMIT_STRING dc3c55f092c96fb3f1dcdff84e6a99f947ea4165) # default SDK tested commit
       if(DEFINED ENV{AZURE_SDK_VCPKG_COMMIT})
         message("AZURE_SDK_VCPKG_COMMIT is defined. Using that instead of the default.")
         set(VCPKG_COMMIT_STRING "$ENV{AZURE_SDK_VCPKG_COMMIT}") # default SDK tested commit
@@ -26,15 +27,15 @@ macro(az_vcpkg_integrate)
       include(FetchContent)
       FetchContent_Declare(
           vcpkg
-          GIT_REPOSITORY      https://github.com/milvus-io/vcpkg.git
+          GIT_REPOSITORY      https://github.com/microsoft/vcpkg.git
           GIT_TAG             ${VCPKG_COMMIT_STRING}
           )
       FetchContent_GetProperties(vcpkg)
       # make sure to pull vcpkg only once.
-      if(NOT vcpkg_POPULATED) 
+      if(NOT vcpkg_POPULATED)
           FetchContent_Populate(vcpkg)
       endif()
-      # use the vcpkg source path 
+      # use the vcpkg source path
       set(CMAKE_TOOLCHAIN_FILE "${vcpkg_SOURCE_DIR}/scripts/buildsystems/vcpkg.cmake" CACHE STRING "")
     endif()
   endif()
@@ -43,6 +44,7 @@ macro(az_vcpkg_integrate)
   if(DEFINED ENV{VCPKG_DEFAULT_TRIPLET} AND NOT DEFINED VCPKG_TARGET_TRIPLET)
     set(VCPKG_TARGET_TRIPLET "$ENV{VCPKG_DEFAULT_TRIPLET}" CACHE STRING "")
   endif()
+  message("Vcpkg integrate step - DONE.")
 endmacro()
 
 macro(az_vcpkg_portfile_prep targetName fileName contentToRemove)
@@ -166,4 +168,4 @@ macro(az_vcpkg_export targetName macroNamePart dllImportExportHeaderPath)
 
   # Export all the installs above as package.
   export(PACKAGE "${targetName}-cpp")
-endmacro()
+endmacro()

internal/core/src/storage/azure-blob-storage/test/CMakeLists.txt

@@ -4,4 +4,5 @@
 project(azure-blob-test)
 
 add_executable(azure-blob-test test_azure_blob_chunk_manager.cpp ../AzureBlobChunkManager.cpp)
-target_link_libraries(azure-blob-test PRIVATE Azure::azure-identity Azure::azure-storage-blobs)
+find_package(GTest CONFIG REQUIRED)
+target_link_libraries(azure-blob-test PRIVATE Azure::azure-identity Azure::azure-storage-blobs GTest::gtest)

internal/core/src/storage/azure-blob-storage/test/test_azure_blob_chunk_manager.cpp

@@ -1,7 +1,60 @@
 #include "../AzureBlobChunkManager.h"
+#include <azure/identity/workload_identity_credential.hpp>
+#include <gtest/gtest.h>
 
 using namespace azure;
 
+void
+SetTenantId(const char* value) {
+    setenv("AZURE_TENANT_ID", value, 1);
+}
+void
+SetClientId(const char* value) {
+    setenv("AZURE_CLIENT_ID", value, 1);
+}
+void
+SetTokenFilePath(const char* value) {
+    setenv("AZURE_FEDERATED_TOKEN_FILE", value, 1);
+}
+
+std::string
+GetTenantId() {
+    return std::getenv("AZURE_TENANT_ID");
+}
+std::string
+GetClientId() {
+    return std::getenv("AZURE_CLIENT_ID");
+}
+std::string
+GetTokenFilePath() {
+    return std::getenv("AZURE_FEDERATED_TOKEN_FILE");
+}
+
+class AzureBlobChunkManagerTest : public testing::Test {
+ protected:
+    void
+    SetUp() override {
+    }
+
+    // void TearDown() override {}
+};
+
+TEST(AzureBlobChunkManagerTest, Options) {
+    SetTenantId("tenant_id");
+    SetClientId("client_id");
+    SetTokenFilePath("token_file_path");
+
+    Azure::Identity::WorkloadIdentityCredentialOptions options;
+    Azure::Identity::WorkloadIdentityCredential const cred(options);
+    EXPECT_EQ(cred.GetCredentialName(), "WorkloadIdentityCredential");
+
+    EXPECT_EQ(options.TenantId, GetTenantId());
+    EXPECT_EQ(options.TenantId, "tenant_id");
+    EXPECT_EQ(options.ClientId, GetClientId());
+    EXPECT_EQ(options.AuthorityHost, "https://login.microsoftonline.com/");
+    EXPECT_EQ(options.TokenFilePath, GetTokenFilePath());
+}
+
 void
 print(Azure::Core::Diagnostics::Logger::Level level,
       std::string const& message) {
@@ -10,7 +63,7 @@ print(Azure::Core::Diagnostics::Logger::Level level,
 }
 
 int
-main() {
+main0() {
     const char* containerName = "default";
     const char* blobName = "sample-blob";
     AzureBlobChunkManager::InitLog("info", print);
@@ -59,3 +112,9 @@ main() {
     chunkManager.DeleteBucket(containerName);
     exit(EXIT_SUCCESS);
 }
+
+int
+main(int argc, char** argv) {
+    testing::InitGoogleTest(&argc, argv);
+    return RUN_ALL_TESTS();
+}

internal/core/src/storage/azure-blob-storage/vcpkg.json

@@ -3,6 +3,7 @@
   "version-string": "1.0.0",
   "dependencies": [
     "azure-identity-cpp",
-    "azure-storage-blobs-cpp"
+    "azure-storage-blobs-cpp",
+    "gtest"
   ]
 }

scripts/azure_build.sh

@@ -39,6 +39,14 @@ fi
 AZURE_CMAKE_CMD="cmake -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
 -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \
 ${SOURCE_DIR}"
+
+if [[ -f "/opt/vcpkg/scripts/buildsystems/vcpkg.cmake" ]]; then
+  AZURE_CMAKE_CMD="cmake -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
+  -DCMAKE_TOOLCHAIN_FILE=/opt/vcpkg/scripts/buildsystems/vcpkg.cmake \
+  -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} \
+  ${SOURCE_DIR}"
+fi
+
 echo ${AZURE_CMAKE_CMD}
 ${AZURE_CMAKE_CMD}