diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 78b7a80d79..590e0b1cf1 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -67,7 +67,7 @@ jobs: - name: 'Setup Use USE_ASAN' if: steps.changed-files-cpp.outputs.any_changed == 'true' run: | - echo "useasan=true" >> $GITHUB_ENV + echo "useasan=ON" >> $GITHUB_ENV echo "Setup USE_ASAN to true since cpp file(s) changed" - name: 'Generate CCache Hash' env: diff --git a/Makefile b/Makefile index 9232b8d905..0c49bb1da4 100644 --- a/Makefile +++ b/Makefile @@ -19,11 +19,17 @@ INSTALL_PATH := $(PWD)/bin LIBRARY_PATH := $(PWD)/lib OS := $(shell uname -s) mode = Release -disk_index = OFF -useasan = false -ifeq (${USE_ASAN}, true) -useasan = true + +use_disk_index = OFF +ifdef USE_DISK_INDEX + use_disk_index = ${USE_DISK_INDEX} endif + +use_asan = OFF +ifdef USE_ASAN + use_asan =${USE_ASAN} +endif + use_dynamic_simd = OFF ifdef USE_DYNAMIC_SIMD use_dynamic_simd = ${USE_DYNAMIC_SIMD} @@ -201,19 +207,19 @@ generated-proto: download-milvus-proto build-3rdparty build-cpp: generated-proto @echo "Building Milvus cpp library ..." - @(env bash $(PWD)/scripts/core_build.sh -t ${mode} -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine}) + @(env bash $(PWD)/scripts/core_build.sh -t ${mode} -n ${use_disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine}) build-cpp-gpu: generated-proto @echo "Building Milvus cpp gpu library ... " - @(env bash $(PWD)/scripts/core_build.sh -t ${mode} -g -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine}) + @(env bash $(PWD)/scripts/core_build.sh -t ${mode} -g -n ${use_disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine}) build-cpp-with-unittest: generated-proto @echo "Building Milvus cpp library with unittest ... " - @(env bash $(PWD)/scripts/core_build.sh -t ${mode} -u -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine}) + @(env bash $(PWD)/scripts/core_build.sh -t ${mode} -u -n ${use_disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine}) build-cpp-with-coverage: generated-proto @echo "Building Milvus cpp library with coverage and unittest ..." - @(env bash $(PWD)/scripts/core_build.sh -t ${mode} -a ${useasan} -u -c -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine}) + @(env bash $(PWD)/scripts/core_build.sh -t ${mode} -a ${use_asan} -u -c -n ${use_disk_index} -y ${use_dynamic_simd} ${AZURE_OPTION} -x ${index_engine}) check-proto-product: generated-proto @(env bash $(PWD)/scripts/check_proto_product.sh) @@ -406,7 +412,7 @@ generate-mockery-querycoord: getdeps generate-mockery-querynode: getdeps build-cpp @source $(PWD)/scripts/setenv.sh # setup PKG_CONFIG_PATH - $(INSTALL_PATH)/mockery --name=QueryHook --dir=$(PWD)/internal/querynodev2/optimizers --output=$(PWD)/internal/querynodev2/optimizers --filename=mock_query_hook.go --with-expecter --outpkg=optimizers --structname=MockQueryHook --inpackage + $(INSTALL_PATH)/mockery --name=QueryHook --dir=$(PWD)/internal/querynodev2/optimizers --output=$(PWD)/internal/querynodev2/optimizers --filename=mock_query_hook.go --with-expecter --outpkg=optimizers --structname=MockQueryHook --inpackage $(INSTALL_PATH)/mockery --name=Manager --dir=$(PWD)/internal/querynodev2/cluster --output=$(PWD)/internal/querynodev2/cluster --filename=mock_manager.go --with-expecter --outpkg=cluster --structname=MockManager --inpackage $(INSTALL_PATH)/mockery --name=SegmentManager --dir=$(PWD)/internal/querynodev2/segments --output=$(PWD)/internal/querynodev2/segments --filename=mock_segment_manager.go --with-expecter --outpkg=segments --structname=MockSegmentManager --inpackage $(INSTALL_PATH)/mockery --name=CollectionManager --dir=$(PWD)/internal/querynodev2/segments --output=$(PWD)/internal/querynodev2/segments --filename=mock_collection_manager.go --with-expecter --outpkg=segments --structname=MockCollectionManager --inpackage diff --git a/internal/core/thirdparty/CMakeLists.txt b/internal/core/thirdparty/CMakeLists.txt index 49e2064aaf..98b628d541 100644 --- a/internal/core/thirdparty/CMakeLists.txt +++ b/internal/core/thirdparty/CMakeLists.txt @@ -26,15 +26,7 @@ include(FetchContent) set(FETCHCONTENT_BASE_DIR ${MILVUS_BINARY_DIR}/3rdparty_download) set(FETCHCONTENT_QUIET OFF) -if(CUSTOM_THIRDPARTY_DOWNLOAD_PATH) - set(THIRDPARTY_DOWNLOAD_PATH ${CUSTOM_THIRDPARTY_DOWNLOAD_PATH}) -else() - set(THIRDPARTY_DOWNLOAD_PATH ${CMAKE_BINARY_DIR}/3rdparty_download/download) -endif() -message(STATUS "Thirdparty downloaded file path: ${THIRDPARTY_DOWNLOAD_PATH}") -# ---------------------------------------------------------------------- # Find pthreads - set(THREADS_PREFER_PTHREAD_FLAG ON) find_package(Threads REQUIRED) diff --git a/scripts/core_build.sh b/scripts/core_build.sh index 983b518229..b2f846c552 100755 --- a/scripts/core_build.sh +++ b/scripts/core_build.sh @@ -90,16 +90,11 @@ BUILD_OUTPUT_DIR="${ROOT_DIR}/cmake_build" BUILD_TYPE="Release" BUILD_UNITTEST="OFF" INSTALL_PREFIX="${CPP_SRC_DIR}/output" -MAKE_CLEAN="OFF" BUILD_COVERAGE="OFF" -DB_PATH="/tmp/milvus" -PROFILING="OFF" RUN_CPPLINT="OFF" CUDA_COMPILER=/usr/local/cuda/bin/nvcc GPU_VERSION="OFF" #defaults to CPU version -WITH_PROMETHEUS="ON" CUDA_ARCH="DEFAULT" -CUSTOM_THIRDPARTY_PATH="" EMBEDDED_MILVUS="OFF" BUILD_DISK_ANN="OFF" USE_ASAN="OFF" @@ -109,15 +104,9 @@ INDEX_ENGINE="KNOWHERE" while getopts "p:d:t:s:f:n:i:y:a:x:ulrcghzmebZ" arg; do case $arg in - f) - CUSTOM_THIRDPARTY_PATH=$OPTARG - ;; p) INSTALL_PREFIX=$OPTARG ;; - d) - DB_PATH=$OPTARG - ;; t) BUILD_TYPE=$OPTARG # BUILD_TYPE ;; @@ -128,23 +117,12 @@ while getopts "p:d:t:s:f:n:i:y:a:x:ulrcghzmebZ" arg; do l) RUN_CPPLINT="ON" ;; - r) - if [[ -d ${BUILD_OUTPUT_DIR} ]]; then - MAKE_CLEAN="ON" - fi - ;; c) BUILD_COVERAGE="ON" ;; - z) - PROFILING="ON" - ;; g) GPU_VERSION="ON" ;; - e) - WITH_PROMETHEUS="OFF" - ;; s) CUDA_ARCH=$OPTARG ;; @@ -156,7 +134,7 @@ while getopts "p:d:t:s:f:n:i:y:a:x:ulrcghzmebZ" arg; do ;; a) ENV_VAL=$OPTARG - if [[ ${ENV_VAL} == 'true' ]]; then + if [[ ${ENV_VAL} == 'ON' ]]; then echo "Set USE_ASAN to ON" USE_ASAN="ON" BUILD_TYPE=Debug @@ -178,15 +156,12 @@ while getopts "p:d:t:s:f:n:i:y:a:x:ulrcghzmebZ" arg; do echo " parameter: --f: custom paths of thirdparty downloaded files(default: NULL) -p: install prefix(default: $(pwd)/milvus) -d: db data path(default: /tmp/milvus) -t: build type(default: Debug) -u: building unit test options(default: OFF) -l: run cpplint, clang-format and clang-tidy(default: OFF) --r: remove previous build directory(default: OFF) -c: code coverage(default: OFF) --z: profiling(default: OFF) -g: build GPU version(default: OFF) -e: build without prometheus(default: OFF) -s: build with CUDA arch(default:DEFAULT), for example '-gencode=compute_61,code=sm_61;-gencode=compute_75,code=sm_75' @@ -196,7 +171,7 @@ parameter: -h: help usage: -./core_build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -s \${CUDA_ARCH} -f\${CUSTOM_THIRDPARTY_PATH} [-u] [-l] [-r] [-c] [-z] [-g] [-m] [-e] [-h] [-b] +./core_build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} -s \${CUDA_ARCH} [-u] [-l] [-r] [-c] [-z] [-g] [-m] [-e] [-h] [-b] " exit 0 ;; @@ -241,26 +216,8 @@ source ${ROOT_DIR}/scripts/setenv.sh CMAKE_GENERATOR="Unix Makefiles" -# MSYS system -if [ "$MSYSTEM" == "MINGW64" ] ; then - BUILD_COVERAGE=OFF - PROFILING=OFF - GPU_VERSION=OFF - WITH_PROMETHEUS=OFF - CUDA_ARCH=OFF - - # extra default cmake args for msys - CMAKE_GENERATOR="MSYS Makefiles" - - # clang tools path - export CLANG_TOOLS_PATH=/mingw64/bin - - # using system blas - export OpenBLAS_HOME="$(cygpath -w /mingw64)" -fi - -# UBUNTU system build diskann index -if [ "$OS_NAME" == "ubuntu20.04" ] ; then +# open build diskann index for Linux +if [ "$OS" == "Linux" ] ; then BUILD_DISK_ANN=ON fi @@ -270,13 +227,6 @@ pushd ${BUILD_OUTPUT_DIR} # Force update the variables each time make rebuild_cache >/dev/null 2>&1 - -if [[ ${MAKE_CLEAN} == "ON" ]]; then - echo "Runing make clean in ${BUILD_OUTPUT_DIR} ..." - make clean - exit 0 -fi - CPU_ARCH=$(get_cpu_arch $CPU_TARGET) arch=$(uname -m) @@ -285,16 +235,11 @@ ${CMAKE_EXTRA_ARGS} \ -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ --DOpenBLAS_SOURCE=AUTO \ -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ -DCMAKE_LIBRARY_ARCHITECTURE=${arch} \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ --DMILVUS_DB_PATH=${DB_PATH} \ --DENABLE_CPU_PROFILING=${PROFILING} \ -DMILVUS_GPU_VERSION=${GPU_VERSION} \ --DMILVUS_WITH_PROMETHEUS=${WITH_PROMETHEUS} \ -DMILVUS_CUDA_ARCH=${CUDA_ARCH} \ --DCUSTOM_THIRDPARTY_DOWNLOAD_PATH=${CUSTOM_THIRDPARTY_PATH} \ -DEMBEDDED_MILVUS=${EMBEDDED_MILVUS} \ -DBUILD_DISK_ANN=${BUILD_DISK_ANN} \ -DUSE_ASAN=${USE_ASAN} \