diff --git a/.github/actions/macos-cache-restore/action.yaml b/.github/actions/macos-cache-restore/action.yaml index e515bd9f02..b081717ebd 100644 --- a/.github/actions/macos-cache-restore/action.yaml +++ b/.github/actions/macos-cache-restore/action.yaml @@ -4,27 +4,51 @@ runs: using: "composite" steps: - name: 'Generate CCache Hash' - env: - CORE_HASH: ${{ hashFiles( 'internal/core/**/*.cpp', 'internal/core/**/*.cc', 'internal/core/**/*.c', 'internal/core/**/*.h', 'internal/core/**/*.hpp', 'internal/core/**/CMakeLists.txt') }} run: | + CORE_HASH="" + if [ -d "internal/core" ]; then + # Find all matching files and calculate hash (macOS compatible) + FILES=$(find internal/core -type f \( -name "*.cpp" -o -name "*.cc" -o -name "*.c" -o -name "*.h" -o -name "*.hpp" -o -name "CMakeLists.txt" \) 2>/dev/null | sort) + if [ -n "$FILES" ]; then + CORE_HASH=$(echo "$FILES" | xargs cat 2>/dev/null | shasum -a 256 | cut -d' ' -f1) + fi + fi echo "corehash=${CORE_HASH}" >> $GITHUB_ENV - echo "Set CCache hash to ${CORE_HASH}" + echo "Set CCache hash to ${CORE_HASH:-empty}" shell: bash - name: Mac Cache CCache Volumes uses: actions/cache/restore@v4 with: path: /var/tmp/ccache - key: macos-ccache-${{ env.corehash }} - restore-keys: macos-ccache- + key: macos15-ccache-${{ env.corehash }} + restore-keys: macos15-ccache- - name: Mac Cache Go Mod Volumes + id: go-mod-cache + run: | + GO_MOD_HASH="" + if find . -name "go.sum" -type f | grep -q .; then + GO_MOD_HASH=$(find . -name "go.sum" -type f -exec cat {} \; 2>/dev/null | shasum -a 256 | cut -d' ' -f1) + fi + echo "hash=${GO_MOD_HASH}" >> $GITHUB_OUTPUT + shell: bash + - name: Restore Go Mod Cache uses: actions/cache/restore@v4 with: path: ~/go/pkg/mod - key: macos-go-mod-${{ hashFiles('**/go.sum') }} - restore-keys: macos-go-mod- + key: macos15-go-mod-${{ steps.go-mod-cache.outputs.hash }} + restore-keys: macos15-go-mod- - name: Mac Cache Conan Packages + id: conan-cache + run: | + CONAN_HASH="" + if find internal/core -name "conanfile.*" -type f 2>/dev/null | grep -q .; then + CONAN_HASH=$(find internal/core -name "conanfile.*" -type f -exec cat {} \; 2>/dev/null | shasum -a 256 | cut -d' ' -f1) + fi + echo "hash=${CONAN_HASH}" >> $GITHUB_OUTPUT + shell: bash + - name: Restore Conan Cache uses: actions/cache/restore@v4 with: path: ~/.conan - key: macos-conan-${{ hashFiles('internal/core/conanfile.*') }} - restore-keys: macos-conan- + key: macos15-conan-${{ steps.conan-cache.outputs.hash }} + restore-keys: macos15-conan- diff --git a/.github/actions/macos-cache-save/action.yaml b/.github/actions/macos-cache-save/action.yaml index e7da248414..98a54063be 100644 --- a/.github/actions/macos-cache-save/action.yaml +++ b/.github/actions/macos-cache-save/action.yaml @@ -13,28 +13,49 @@ runs: using: "composite" steps: - name: Generate CCache Hash - env: - CORE_HASH: ${{ hashFiles( 'internal/core/**/*.cpp', 'internal/core/**/*.cc', 'internal/core/**/*.c', 'internal/core/**/*.h', 'internal/core/**/*.hpp', 'internal/core/**/CMakeLists.txt') }} run: | + CORE_HASH="" + if [ -d "internal/core" ]; then + # Find all matching files and calculate hash (macOS compatible) + FILES=$(find internal/core -type f \( -name "*.cpp" -o -name "*.cc" -o -name "*.c" -o -name "*.h" -o -name "*.hpp" -o -name "CMakeLists.txt" \) 2>/dev/null | sort) + if [ -n "$FILES" ]; then + CORE_HASH=$(echo "$FILES" | xargs cat 2>/dev/null | shasum -a 256 | cut -d' ' -f1) + fi + fi echo "corehash=${CORE_HASH}" >> $GITHUB_ENV - echo "Set CCache hash to ${CORE_HASH}" + echo "Set CCache hash to ${CORE_HASH:-empty}" shell: bash - name: Mac Cache CCache Volumes uses: actions/cache/save@v4 with: path: /var/tmp/ccache - key: macos-ccache-${{ env.corehash }} - restore-keys: macos-ccache- + key: macos15-ccache-${{ env.corehash }} + - name: Generate Go Mod Hash + id: go-mod-hash + run: | + GO_MOD_HASH="" + if find . -name "go.sum" -type f | grep -q .; then + GO_MOD_HASH=$(find . -name "go.sum" -type f -exec cat {} \; 2>/dev/null | shasum -a 256 | cut -d' ' -f1) + fi + echo "hash=${GO_MOD_HASH}" >> $GITHUB_OUTPUT + shell: bash - name: Mac Cache Go Mod Volumes uses: actions/cache/save@v4 with: path: ~/go/pkg/mod - key: macos-go-mod-${{ hashFiles('**/go.sum') }} - restore-keys: macos-go-mod- + key: macos15-go-mod-${{ steps.go-mod-hash.outputs.hash }} + - name: Generate Conan Hash + id: conan-hash + run: | + CONAN_HASH="" + if find internal/core -name "conanfile.*" -type f 2>/dev/null | grep -q .; then + CONAN_HASH=$(find internal/core -name "conanfile.*" -type f -exec cat {} \; 2>/dev/null | shasum -a 256 | cut -d' ' -f1) + fi + echo "hash=${CONAN_HASH}" >> $GITHUB_OUTPUT + shell: bash - name: Mac Cache Conan Packages uses: actions/cache/save@v4 with: path: ~/.conan - key: macos-conan-${{ hashFiles('internal/core/conanfile.*') }} - restore-keys: macos-conan- + key: macos15-conan-${{ steps.conan-hash.outputs.hash }} diff --git a/.github/mergify.yml b/.github/mergify.yml index 627e3f489c..2171d0bc51 100644 --- a/.github/mergify.yml +++ b/.github/mergify.yml @@ -19,6 +19,21 @@ misc: - when_integration_unit_test_success: &WHEN_INTEGRATION_UNIT_TEST_SUCCESS - 'status-success=integration-test' - 'status-success=Integration Test' + - 'status-success=ci-v2/integration-test' + - when_e2e_test_success: &WHEN_E2E_TEST_SUCCESS + - 'status-success=cpu-e2e' + - 'status-success=ci-v2/e2e-default' + - when_build_success: &WHEN_BUILD_SUCCESS + - 'status-success=Build and test AMD64 Ubuntu 22.04' + - 'status-success=ci-v2/build' + - when_code_check_ubuntu_success: &WHEN_CODE_CHECK_UBUNTU_SUCCESS + - 'status-success=Code Checker AMD64 Ubuntu 22.04' + - 'status-success=ci-v2/code-check' + - when_code_check_macos_success: &WHEN_CODE_CHECK_MACOS_SUCCESS + - 'status-success=Code Checker MacOS 13' + - 'status-success=Code Checker MacOS' + + # Branch configurations - branch: &BRANCHES # In this pull request, the changes are based on the master branch - &MASTER_BRANCH base=master @@ -64,10 +79,9 @@ pull_request_rules: - or: *WHEN_CPP_UNIT_TEST_SUCCESS - or: *WHEN_GO_UNIT_TEST_SUCCESS - or: *WHEN_INTEGRATION_UNIT_TEST_SUCCESS - - 'status-success=Code Checker AMD64 Ubuntu 22.04' - - 'status-success=Code Checker MacOS 13' - # - 'status-success=Code Checker Amazonlinux 2023' - - 'status-success=cpu-e2e' + - or: *WHEN_E2E_TEST_SUCCESS + - or: *WHEN_CODE_CHECK_UBUNTU_SUCCESS + - or: *WHEN_CODE_CHECK_MACOS_SUCCESS # - 'status-success=codecov/patch' # - 'status-success=codecov/project' actions: @@ -82,9 +96,9 @@ pull_request_rules: - or: *WHEN_CPP_UNIT_TEST_SUCCESS - or: *WHEN_GO_UNIT_TEST_SUCCESS - or: *WHEN_INTEGRATION_UNIT_TEST_SUCCESS - - 'status-success=Code Checker MacOS 13' - # - 'status-success=Code Checker CentOS 7' - - 'status-success=cpu-e2e' + - or: *WHEN_E2E_TEST_SUCCESS + - or: *WHEN_CODE_CHECK_UBUNTU_SUCCESS + - or: *WHEN_CODE_CHECK_MACOS_SUCCESS # - 'status-success=codecov/patch' # - 'status-success=codecov/project' actions: @@ -128,10 +142,10 @@ pull_request_rules: conditions: # branch condition: in this pull request, the changes are based on any branch referenced by BRANCHES - or: *BRANCHES - - or: *Build_AND_TEST_STATUS_SUCESS_ON_UBUNTU_20_OR_UBUNTU_22 - - 'status-success=Code Checker AMD64 Ubuntu 22.04' - - 'status-success=Code Checker MacOS 13' - # - 'status-success=Code Checker Amazonlinux 2023' + - *only_go_unittest_files + - or: *WHEN_BUILD_SUCCESS + - or: *WHEN_CODE_CHECK_UBUNTU_SUCCESS + - or: *WHEN_CODE_CHECK_MACOS_SUCCESS - or: *WHEN_GO_UNIT_TEST_SUCCESS - or: - -files~=^(?!pkg\/.*_test\.go).*$ @@ -176,6 +190,126 @@ pull_request_rules: add: - ci-passed + - name: Test passed for skip e2e when source code changed + conditions: + - or: *BRANCHES + - or: *WHEN_BUILD_SUCCESS + - title~=\[skip e2e\] + - or: *WHEN_CPP_UNIT_TEST_SUCCESS + - or: *WHEN_GO_UNIT_TEST_SUCCESS + - or: *WHEN_INTEGRATION_UNIT_TEST_SUCCESS + - or: *WHEN_CODE_CHECK_UBUNTU_SUCCESS + - or: *WHEN_CODE_CHECK_MACOS_SUCCESS + - *source_code_files + actions: + label: + add: + - ci-passed + + # ========================================================================== + # CI FAILURE HANDLING + # Remove ci-passed labels when tests fail + # ========================================================================== + + # ========================================================================== + # CONSOLIDATED MIGRATION FAILURE HANDLING + # Handle failure scenarios during CI system migration with optimized rules + # ========================================================================== + + - name: Remove ci-passed when any CI system fails during migration + conditions: + - or: + - *MASTER_BRANCH + - *2X_BRANCH + - label!=manual-pass + - *source_code_files + # Comprehensive failure logic: remove ci-passed when any required system fails + - or: + # Build systems: remove if no success AND has failure + - and: + - not: + or: + - status-success = Build and test AMD64 Ubuntu 22.04 + - status-success = ci-v2/build + - or: + - status-failure = Build and test AMD64 Ubuntu 22.04 + - status-failure = ci-v2/build + # Go unit tests: remove if no success AND has failure + - and: + - not: + or: + - status-success = UT for Go + - status-success = ci-v2/ut-go + - or: + - status-failure = UT for Go + - status-failure = ci-v2/ut-go + # Integration tests: remove if no success AND has failure + - and: + - not: + or: + - status-success = Integration Test + - status-success = ci-v2/integration-test + - or: + - status-failure = Integration Test + - status-failure = ci-v2/integration-test + # Cpp unit tests: remove if no success AND has failure (only when cpp files changed) + - and: + - *morethan_go_unittest_files + - not: + or: + - status-success = cpp-unit-test + - status-success = ci-v2/ut-cpp + - or: + - status-failure = cpp-unit-test + - status-failure = ci-v2/ut-cpp + # Code checker Ubuntu: remove if no success AND has failure + - and: + - not: + or: + - status-success = Code Checker AMD64 Ubuntu 22.04 + - status-success = ci-v2/code-check + - or: + - status-failure = Code Checker AMD64 Ubuntu 22.04 + - status-failure = ci-v2/code-check + # Code checker MacOS: remove if no success AND has failure + - and: + - not: + or: + - status-success = Code Checker MacOS 13 + - status-success = Code Checker MacOS + - or: + - status-failure = Code Checker MacOS 13 + - status-failure = Code Checker MacOS + # E2E tests: regular case - remove if no success AND has failure + - and: + - not: + or: + - status-success = cpu-e2e + - status-success = ci-v2/e2e-default + - or: + - status-failure = cpu-e2e + - status-failure = ci-v2/e2e-default + # E2E tests: required case - remove if no success AND has failure (when not skipped) + - and: + - -title~=\[skip e2e\] + - files~=^(?!(.*_test\.go|.*\.md|\.github\/mergify\.yml)).*$ + - not: + or: + - status-success = cpu-e2e + - status-success = ci-v2/e2e-default + - or: + - status-failure = cpu-e2e + - status-failure = ci-v2/e2e-default + actions: + label: + remove: + - ci-passed + + # ========================================================================== + # PR VALIDATION AND BLOCKING RULES + # Ensure PRs meet project standards before merging + # ========================================================================== + - name: Blocking PR if missing a related issue or doesn't have kind/enhancement label conditions: # branch condition: in this pull request, the changes are based on any branch referenced by BRANCHES diff --git a/.github/workflows/mac.yaml b/.github/workflows/mac.yaml index ab07659998..281b32cd9c 100644 --- a/.github/workflows/mac.yaml +++ b/.github/workflows/mac.yaml @@ -32,9 +32,9 @@ concurrency: jobs: mac: - name: Code Checker MacOS 13 - runs-on: macos-13 - timeout-minutes: 300 + name: Code Checker MacOS + runs-on: macos-15-intel + timeout-minutes: 480 steps: - name: Checkout uses: actions/checkout@v2 @@ -44,18 +44,23 @@ jobs: python-version: '<3.12' - name: Install CMake 3.31.8 run: | - cmake --version - which cmake - brew uninstall cmake -f + cmake --version || true + which cmake || true + if ! brew list cmake &>/dev/null; then + brew uninstall cmake -f + fi curl -LO https://github.com/Kitware/CMake/releases/download/v3.31.8/cmake-3.31.8-macos-universal.tar.gz tar -zxf cmake-3.31.8-macos-universal.tar.gz -C /tmp/ sudo mv /tmp/cmake-3.31.8-macos-universal/CMake.app /Applications/ + sudo rm -f /usr/local/bin/cmake /usr/local/bin/cpack /usr/local/bin/ctest + sudo rm -f /opt/homebrew/bin/cmake /opt/homebrew/bin/cpack /opt/homebrew/bin/ctest echo -e "\n=== Creating symlinks for CMake ===" sudo ln -s /Applications/CMake.app/Contents/bin/cmake /usr/local/bin/cmake sudo ln -s /Applications/CMake.app/Contents/bin/cpack /usr/local/bin/cpack sudo ln -s /Applications/CMake.app/Contents/bin/ctest /usr/local/bin/ctest CMAKE_MODULE_DIR="/Applications/CMake.app/Contents/share/cmake-3.31" sudo ln -s "$CMAKE_MODULE_DIR" /usr/local/share/cmake-3.31 + export PATH="/usr/local/bin:$PATH" cmake --version which cmake - name: Setup Go environment @@ -76,12 +81,14 @@ jobs: if [[ ! -d "/var/tmp/ccache" ]];then mkdir -p /var/tmp/ccache fi - ls -alh /var/tmp/ccache brew install libomp ninja openblas ccache pkg-config - pip3 install conan==1.64.1 - if [[ ! -d "/usr/local/opt/llvm" ]]; then - ln -s /usr/local/opt/llvm@14 /usr/local/opt/llvm + BREW_PREFIX=$(brew --prefix) + # Install llvm@17 explicitly as a stable version + brew install llvm@17 + if [[ ! -d "${BREW_PREFIX}/opt/llvm" ]]; then + ln -s ${BREW_PREFIX}/opt/llvm@17 ${BREW_PREFIX}/opt/llvm fi + pip3 install conan==1.64.1 make milvus - name: Upload Cmake log uses: actions/upload-artifact@v4