enhance: enable asan for cpp unittest (#37041)

https://github.com/milvus-io/milvus/issues/35854

Signed-off-by: chyezh <chyezh@outlook.com>
Co-authored-by: chyezh <chyezh@outlook.com>

.github/workflows/main.yaml

@@ -112,6 +112,8 @@ jobs:
           kind: 'cpp'
 
   UT-Cpp:
+    # skip the UT-Cpp job in github workflow, it run in jenkins now see: UT-CPP.groovy
+    if: false 
     name: UT for Cpp
     needs: Build
     runs-on: ubuntu-latest
@@ -268,7 +270,7 @@ jobs:
 
   codecov:
     name: Upload Code Coverage
-    needs: [UT-Cpp, UT-Go, integration-test]
+    needs: [UT-Go, integration-test]
     runs-on: ubuntu-latest
     timeout-minutes: 5
     steps:
@@ -276,10 +278,6 @@ jobs:
         uses: actions/checkout@v2
         with:
           fetch-depth: 0
-      - name: Download Cpp code coverage results
-        uses: actions/download-artifact@v4.1.0
-        with:
-          name: cpp-result
       - name: Download Go code coverage results
         uses: actions/download-artifact@v4.1.0
         with:
@@ -297,7 +295,7 @@ jobs:
         id: upload_cov
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./go_coverage.txt,./lcov_output.info,./it_coverage.txt
+          files: ./go_coverage.txt,./it_coverage.txt
           name: ubuntu-20.04-unittests
           fail_ci_if_error: true
           disable_safe_directory: true
@@ -307,7 +305,7 @@ jobs:
         id: retry_upload_cov
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
-          files: ./go_coverage.txt,./lcov_output.info,./it_coverage.txt
+          files: ./go_coverage.txt,./it_coverage.txt
           name: ubuntu-20.04-unittests
           fail_ci_if_error: true
           disable_safe_directory: true

Makefile

@@ -30,8 +30,8 @@ endif
 use_asan = OFF
 ifdef USE_ASAN
 	use_asan =${USE_ASAN}
-	CGO_LDFLAGS := $(shell go env CGO_LDFLAGS) -fsanitize=address -fno-omit-frame-pointer
+	CGO_LDFLAGS := $(shell go env CGO_LDFLAGS) -fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address
-	CGO_CFLAGS := $(shell go env CGO_CFLAGS) -fsanitize=address -fno-omit-frame-pointer
+	CGO_CFLAGS := $(shell go env CGO_CFLAGS) -fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address
 	MILVUS_GO_BUILD_TAGS := $(MILVUS_GO_BUILD_TAGS),use_asan
 endif
 

ci/jenkins/UT-CPP.groovy

@@ -1,4 +1,4 @@
-@Library('jenkins-shared-library@tekton') _
+@Library('jenkins-shared-library@v0.62.0') _
 
 def pod = libraryResource 'io/milvus/pod/tekton-4am.yaml'
 def milvus_helm_chart_version = '4.2.8'
@@ -38,7 +38,7 @@ pipeline {
                                               gitMode: gitMode ,
                                               gitBaseRef: gitBaseRef,
                                               pullRequestNumber: "$env.CHANGE_ID",
-                                              make_cmd: "make clean && make USE_ASAN=OFF build-cpp-with-coverage",
+                                              make_cmd: "make clean && make USE_ASAN=ON build-cpp-with-coverage",
                                               test_entrypoint: "./scripts/run_cpp_codecov.sh",
                                               codecov_files: "./lcov_output.info,./it_coverage.txt"
                     }

internal/core/CMakeLists.txt

@@ -131,8 +131,8 @@ if (LINUX OR MSYS)
                   )
     if (USE_ASAN STREQUAL "ON") 
         message( STATUS "Building Milvus Core Using AddressSanitizer")
-        add_compile_options(-fno-omit-frame-pointer -fsanitize=address)
+        add_compile_options(-fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address)
-        add_link_options(-fno-omit-frame-pointer -fsanitize=address)
+        add_link_options(-fno-stack-protector -fno-omit-frame-pointer -fno-var-tracking -fsanitize=address)
     endif()
     if (CMAKE_BUILD_TYPE STREQUAL "Release")
         append_flags( CMAKE_CXX_FLAGS

internal/core/src/segcore/vector_index_c.cpp

@@ -27,7 +27,7 @@ GetIndexListSize() {
 
 void
 GetIndexFeatures(void* index_key_list, uint64_t* index_feature_list) {
-    auto features = knowhere::IndexFactory::Instance().GetIndexFeatures();
+    auto& features = knowhere::IndexFactory::Instance().GetIndexFeatures();
     int idx = 0;
 
     const char** index_keys = (const char**)index_key_list;

internal/core/unittest/test_storage.cpp

@@ -38,7 +38,8 @@ get_azure_storage_config() {
         "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/"
         "K1SZFPTOtr/KBHBeksoGMGw==";
 
-    return CStorageConfig{endpoint,
+    return CStorageConfig{
+        endpoint,
         bucketName.c_str(),
         accessKey,
         accessValue,
@@ -52,7 +53,9 @@ get_azure_storage_config() {
         "",
         false,
         false,
-                          30000};
+        30000,
+        "",
+    };
 }
 
 class StorageTest : public testing::Test {
@@ -93,7 +96,9 @@ TEST_F(StorageTest, GetLocalUsedSize) {
 
 TEST_F(StorageTest, InitRemoteChunkManagerSingleton) {
     CStorageConfig storageConfig = get_azure_storage_config();
-    InitRemoteChunkManagerSingleton(storageConfig);
+    auto status = InitRemoteChunkManagerSingleton(storageConfig);
+    EXPECT_STREQ(status.error_msg, "");
+    EXPECT_EQ(status.error_code, Success);
     auto rcm =
         RemoteChunkManagerSingleton::GetInstance().GetRemoteChunkManager();
     EXPECT_EQ(rcm->GetRootPath(), "/tmp/milvus/remote_data");

scripts/core_build.sh

@@ -138,7 +138,6 @@ while getopts "p:d:t:s:f:n:i:y:a:x:o:ulrcghzmebZ" arg; do
     if [[ ${ENV_VAL} == 'ON' ]]; then
         echo "Set USE_ASAN to ON"
         USE_ASAN="ON"
-        BUILD_TYPE=Debug
     fi
     ;;
   y)

scripts/install_milvus.sh

@@ -24,8 +24,6 @@ cp $PWD"/internal/core/output/lib/"*.dylib* "$LIBRARY_PATH" 2>/dev/null || true
 cp $PWD"/internal/core/output/lib/"*.so* "$LIBRARY_PATH" || true
 cp $PWD"/internal/core/output/lib64/"*.so* "$LIBRARY_PATH" 2>/dev/null || true
 
-if [ "$USE_ASAN" == "ON" ]; then
+for LIB_PATH in $(ldd ./bin/milvus | grep -E '(asan|atomic)' | awk '{print $3}'); do
-    for LIB_PATH in $(ldconfig -p | grep -E '(asan|atomic)' | awk '{print $NF}'); do
     cp "$LIB_PATH" "$LIBRARY_PATH" 2>/dev/null
-    done
+done
-fi

scripts/run_cpp_codecov.sh

@@ -21,11 +21,11 @@ set -e
 
 SOURCE="${BASH_SOURCE[0]}"
 while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
-  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+    DIR="$(cd -P "$(dirname "$SOURCE")" && pwd)"
     SOURCE="$(readlink "$SOURCE")"
     [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
 done
-ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )"
+ROOT_DIR="$(cd -P "$(dirname "$SOURCE")/.." && pwd)"
 source ${ROOT_DIR}/scripts/setenv.sh
 
 MILVUS_CORE_DIR="${ROOT_DIR}/internal/core"
@@ -59,13 +59,18 @@ if [ $? -ne 0 ]; then
     exit -1
 fi
 # starting the timer
-beginTime=`date +%s`
+beginTime=$(date +%s)
 
 # run unittest
-for test in `ls ${MILVUS_CORE_UNITTEST_DIR}`; do
+for test in $(ls ${MILVUS_CORE_UNITTEST_DIR}); do
     echo "Running cpp unittest: ${MILVUS_CORE_UNITTEST_DIR}/$test"
     # run unittest
+    if [ -n "$MILVUS_ENABLE_ASAN_LIB" ]; then
+        echo "ASAN is enabled with env MILVUS_ENABLE_ASAN_LIB, set {$MILVUS_ENABLE_ASAN_LIB} at the front of LD_PRELOAD"
+        LD_PRELOAD="$MILVUS_ENABLE_ASAN_LIB:$LD_PRELOAD" ${MILVUS_CORE_UNITTEST_DIR}/${test}
+    else
         ${MILVUS_CORE_UNITTEST_DIR}/${test}
+    fi
     if [ $? -ne 0 ]; then
         echo ${args}
         echo "${MILVUS_CORE_UNITTEST_DIR}/${test} run failed"
@@ -94,7 +99,6 @@ ${LCOV_CMD} -r "${FILE_INFO_COMBINE}" -o "${FILE_INFO_OUTPUT}" \
 ${LCOV_GEN_CMD} ${FILE_INFO_OUTPUT} --output-directory ${DIR_LCOV_OUTPUT}/
 echo "Generate cpp code coverage report to ${DIR_LCOV_OUTPUT}"
 
+endTime=$(date +%s)
 
-endTime=`date +%s`
+echo "Total time for cpp unittest:" $(($endTime - $beginTime)) "s"
-
-echo "Total time for cpp unittest:" $(($endTime-$beginTime)) "s"

scripts/setenv.sh

@@ -36,6 +36,13 @@ unameOut="$(uname -s)"
 
 case "${unameOut}" in
     Linux*)
+      # check if use asan.
+      MILVUS_ENABLE_ASAN_LIB=$(ldd $ROOT_DIR/internal/core/output/lib/libmilvus_core.so | grep asan | awk '{print $3}')
+      if [ -n "$MILVUS_ENABLE_ASAN_LIB" ]; then
+          echo "Enable ASAN With ${MILVUS_ENABLE_ASAN_LIB}"
+          export MILVUS_ENABLE_ASAN_LIB="$MILVUS_ENABLE_ASAN_LIB"
+      fi
+
       LIBJEMALLOC=$PWD/internal/core/output/lib/libjemalloc.so
       if test -f "$LIBJEMALLOC"; then
         export LD_PRELOAD="$LIBJEMALLOC"