From b59ba813492fc873e87d36ebacf370c1b8ef5ed9 Mon Sep 17 00:00:00 2001 From: yellow-shine Date: Tue, 6 Aug 2024 10:12:27 +0800 Subject: [PATCH] feat: [skip e2e] effective way to use cache (#35233) issue: https://github.com/milvus-io/milvus/issues/34876 https://github.com/milvus-io/milvus/pull/35109 --------- Signed-off-by: Yellow Shine --- .github/actions/cache-restore/action.yaml | 49 +++++++++++++++++++++++ .github/actions/cache-save/action.yaml | 49 +++++++++++++++++++++++ .github/workflows/code-checker.yaml | 15 ++++++- .github/workflows/main.yaml | 26 ++++++++++-- 4 files changed, 134 insertions(+), 5 deletions(-) create mode 100644 .github/actions/cache-restore/action.yaml create mode 100644 .github/actions/cache-save/action.yaml diff --git a/.github/actions/cache-restore/action.yaml b/.github/actions/cache-restore/action.yaml new file mode 100644 index 0000000000..2c99a22257 --- /dev/null +++ b/.github/actions/cache-restore/action.yaml @@ -0,0 +1,49 @@ +name: 'Milvus Cache' +description: '' +inputs: + os: + description: 'OS name' + required: true + default: 'ubuntu20.04' + kind: + description: 'Cache kind' + required: false + default: 'all' +runs: + using: "composite" + steps: + - name: Generate CCache Hash + env: + CORE_HASH: ${{ hashFiles( 'internal/core/**/*.cpp', 'internal/core/**/*.cc', 'internal/core/**/*.c', 'internal/core/**/*.h', 'internal/core/**/*.hpp', 'internal/core/**/CMakeLists.txt') }} + run: | + echo "corehash=${CORE_HASH}" >> $GITHUB_ENV + echo "Set CCache hash to ${CORE_HASH}" + shell: bash + - name: Cache CCache Volumes + if: ${{ inputs.kind == 'all' || inputs.kind == 'cpp' }} + uses: actions/cache/restore@v4 + with: + path: .docker/amd64-${{ inputs.os }}-ccache + key: ${{ inputs.os }}-ccache-${{ env.corehash }} + restore-keys: ${{ inputs.os }}-ccache- + - name: Cache Conan Packages + if: ${{ inputs.kind == 'all' || inputs.kind == 'cpp' }} + uses: actions/cache/restore@v4 + with: + path: .docker/amd64-${{ inputs.os }}-conan + key: ${{ inputs.os }}-conan-${{ hashFiles('internal/core/conanfile.*') }} + restore-keys: ${{ inputs.os }}-conan- + - name: Cache Third Party + if: ${{ inputs.kind == 'all' || inputs.kind == 'thirdparty' }} + uses: actions/cache/restore@v4 + with: + path: .docker/thirdparty + key: ${{ inputs.os }}-thirdparty-${{ hashFiles('internal/core/thirdparty/**') }} + restore-keys: ${{ inputs.os }}-thirdparty- + - name: Cache Go Mod Volumes + if: ${{ inputs.kind == 'all' || inputs.kind == 'go' }} + uses: actions/cache/restore@v4 + with: + path: .docker/amd64-${{ inputs.os }}-go-mod + key: ${{ inputs.os }}-go-mod-${{ hashFiles('go.sum, */go.sum') }} + restore-keys: ${{ inputs.os }}-go-mod- diff --git a/.github/actions/cache-save/action.yaml b/.github/actions/cache-save/action.yaml new file mode 100644 index 0000000000..89d973b32a --- /dev/null +++ b/.github/actions/cache-save/action.yaml @@ -0,0 +1,49 @@ +name: 'Milvus Cache' +description: '' +inputs: + os: + description: 'OS name' + required: true + default: 'ubuntu20.04' + kind: + description: 'Cache kind' + required: false + default: 'all' +runs: + using: "composite" + steps: + - name: Generate CCache Hash + env: + CORE_HASH: ${{ hashFiles( 'internal/core/**/*.cpp', 'internal/core/**/*.cc', 'internal/core/**/*.c', 'internal/core/**/*.h', 'internal/core/**/*.hpp', 'internal/core/**/CMakeLists.txt') }} + run: | + echo "corehash=${CORE_HASH}" >> $GITHUB_ENV + echo "Set CCache hash to ${CORE_HASH}" + shell: bash + - name: Cache CCache Volumes + if: ${{ inputs.kind == 'all' || inputs.kind == 'cpp' }} + uses: actions/cache/save@v4 + with: + path: .docker/amd64-${{ inputs.os }}-ccache + key: ${{ inputs.os }}-ccache-${{ env.corehash }} + restore-keys: ${{ inputs.os }}-ccache- + - name: Cache Conan Packages + if: ${{ inputs.kind == 'all' || inputs.kind == 'cpp' }} + uses: actions/cache/save@v4 + with: + path: .docker/amd64-${{ inputs.os }}-conan + key: ${{ inputs.os }}-conan-${{ hashFiles('internal/core/conanfile.*') }} + restore-keys: ${{ inputs.os }}-conan- + - name: Cache Third Party + if: ${{ inputs.kind == 'all' || inputs.kind == 'thirdparty' }} + uses: actions/cache/save@v4 + with: + path: .docker/thirdparty + key: ${{ inputs.os }}-thirdparty-${{ hashFiles('internal/core/thirdparty/**') }} + restore-keys: ${{ inputs.os }}-thirdparty- + - name: Cache Go Mod Volumes + if: ${{ inputs.kind == 'all' || inputs.kind == 'go' }} + uses: actions/cache/save@v4 + with: + path: .docker/amd64-${{ inputs.os }}-go-mod + key: ${{ inputs.os }}-go-mod-${{ hashFiles('go.sum, */go.sum') }} + restore-keys: ${{ inputs.os }}-go-mod- diff --git a/.github/workflows/code-checker.yaml b/.github/workflows/code-checker.yaml index 9d67f0bb3e..f8d8ef2bbd 100644 --- a/.github/workflows/code-checker.yaml +++ b/.github/workflows/code-checker.yaml @@ -51,7 +51,7 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Download Caches - uses: ./.github/actions/cache + uses: ./.github/actions/cache-restore with: os: 'ubuntu20.04' - name: Code Check @@ -59,6 +59,11 @@ jobs: OS_NAME: 'ubuntu20.04' run: | ./build/builder.sh /bin/bash -c "make check-proto-product && make verifiers" + - name: Save Caches + uses: ./.github/actions/cache-save + if: github.event_name != 'pull_request' + with: + os: 'ubuntu20.04' amazonlinux: name: Code Checker Amazonlinux 2023 @@ -78,10 +83,16 @@ jobs: - name: Checkout uses: actions/checkout@v2 - name: Download Caches - uses: ./.github/actions/cache + uses: ./.github/actions/cache-restore with: os: 'amazonlinux2023' - name: Code Check run: | sed -i 's/ubuntu20.04/amazonlinux2023/g' .env ./build/builder.sh /bin/bash -c "make install" + - name: Save Caches + uses: ./.github/actions/cache-save + if: github.event_name != 'pull_request' + with: + os: 'amazonlinux2023' + diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 54cbf2a477..63c016ffa9 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -80,7 +80,7 @@ jobs: echo "useasan=ON" >> $GITHUB_ENV echo "Setup USE_ASAN to true since cpp file(s) changed" - name: Download Caches - uses: ./.github/actions/cache + uses: ./.github/actions/cache-restore with: os: 'ubuntu20.04' kind: 'cpp' @@ -94,6 +94,13 @@ jobs: with: name: code path: code.zip + - name: Save Caches + uses: ./.github/actions/cache-save + if: github.event_name != 'pull_request' + with: + os: 'ubuntu20.04' + kind: 'cpp' + UT-Cpp: name: UT for Cpp needs: Build @@ -117,7 +124,7 @@ jobs: unzip code.zip rm code.zip - name: Download Caches - uses: ./.github/actions/cache + uses: ./.github/actions/cache-restore with: os: 'ubuntu20.04' kind: 'cpp' @@ -142,6 +149,12 @@ jobs: ./lcov_output.info *.info *.out + - name: Save Caches + uses: ./.github/actions/cache-save + if: github.event_name != 'pull_request' + with: + os: 'ubuntu20.04' + kind: 'cpp' UT-Go: name: UT for Go needs: Build @@ -165,7 +178,7 @@ jobs: unzip code.zip rm code.zip - name: Download Caches - uses: ./.github/actions/cache + uses: ./.github/actions/cache-restore with: os: 'ubuntu20.04' kind: 'go' @@ -234,6 +247,13 @@ jobs: ./it_coverage.txt *.info *.out + - name: Save Caches + uses: ./.github/actions/cache-save + if: github.event_name != 'pull_request' + with: + os: 'ubuntu20.04' + kind: 'go' + codecov: name: Upload Code Coverage needs: [UT-Cpp, UT-Go, integration-test]