diff --git a/ci/jenkins/scripts/build.sh b/ci/jenkins/scripts/build.sh deleted file mode 100755 index 2ccdf4a618..0000000000 --- a/ci/jenkins/scripts/build.sh +++ /dev/null @@ -1,142 +0,0 @@ -#!/bin/bash - -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - -CMAKE_BUILD_DIR="${SCRIPTS_DIR}/../../../core/cmake_build" -BUILD_TYPE="Debug" -BUILD_UNITTEST="OFF" -INSTALL_PREFIX="/opt/milvus" -BUILD_COVERAGE="OFF" -DB_PATH="/opt/milvus" -PROFILING="OFF" -USE_JFROG_CACHE="OFF" -RUN_CPPLINT="OFF" -CUSTOMIZATION="OFF" # default use ori faiss -CUDA_COMPILER=/usr/local/cuda/bin/nvcc - -CUSTOMIZED_FAISS_URL="${FAISS_URL:-NONE}" -wget -q --method HEAD ${CUSTOMIZED_FAISS_URL} -if [ $? -eq 0 ]; then - CUSTOMIZATION="ON" -else - CUSTOMIZATION="OFF" -fi - -while getopts "o:d:t:ulcgjhx" arg -do - case $arg in - o) - INSTALL_PREFIX=$OPTARG - ;; - d) - DB_PATH=$OPTARG - ;; - t) - BUILD_TYPE=$OPTARG # BUILD_TYPE - ;; - u) - echo "Build and run unittest cases" ; - BUILD_UNITTEST="ON"; - ;; - l) - RUN_CPPLINT="ON" - ;; - c) - BUILD_COVERAGE="ON" - ;; - g) - PROFILING="ON" - ;; - j) - USE_JFROG_CACHE="ON" - ;; - x) - CUSTOMIZATION="OFF" # force use ori faiss - ;; - h) # help - echo " - -parameter: --o: install prefix(default: /opt/milvus) --d: db data path(default: /opt/milvus) --t: build type(default: Debug) --u: building unit test options(default: OFF) --l: run cpplint, clang-format and clang-tidy(default: OFF) --c: code coverage(default: OFF) --g: profiling(default: OFF) --j: use jfrog cache build directory(default: OFF) --h: help - -usage: -./build.sh -p \${INSTALL_PREFIX} -t \${BUILD_TYPE} [-u] [-l] [-r] [-c] [-g] [-j] [-h] - " - exit 0 - ;; - ?) - echo "ERROR! unknown argument" - exit 1 - ;; - esac -done - -if [[ ! -d ${CMAKE_BUILD_DIR} ]]; then - mkdir ${CMAKE_BUILD_DIR} -fi - -cd ${CMAKE_BUILD_DIR} - -# remove make cache since build.sh -l use default variables -# force update the variables each time -make rebuild_cache - -CMAKE_CMD="cmake \ --DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ --DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} --DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ --DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ --DBUILD_COVERAGE=${BUILD_COVERAGE} \ --DMILVUS_DB_PATH=${DB_PATH} \ --DMILVUS_ENABLE_PROFILING=${PROFILING} \ --DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ --DCUSTOMIZATION=${CUSTOMIZATION} \ --DFAISS_URL=${CUSTOMIZED_FAISS_URL} \ -.." -echo ${CMAKE_CMD} -${CMAKE_CMD} - -if [[ ${RUN_CPPLINT} == "ON" ]]; then - # cpplint check - make lint - if [ $? -ne 0 ]; then - echo "ERROR! cpplint check failed" - exit 1 - fi - echo "cpplint check passed!" - - # clang-format check - make check-clang-format - if [ $? -ne 0 ]; then - echo "ERROR! clang-format check failed" - exit 1 - fi - echo "clang-format check passed!" - -# # clang-tidy check -# make check-clang-tidy -# if [ $? -ne 0 ]; then -# echo "ERROR! clang-tidy check failed" -# rm -f CMakeCache.txt -# exit 1 -# fi -# echo "clang-tidy check passed!" -else - # compile and build - make -j8 || exit 1 - make install || exit 1 -fi diff --git a/ci/jenkins/step/build.groovy b/ci/jenkins/step/build.groovy index 14d0414f4f..5b3a46818d 100644 --- a/ci/jenkins/step/build.groovy +++ b/ci/jenkins/step/build.groovy @@ -1,8 +1,7 @@ timeout(time: 60, unit: 'MINUTES') { - dir ("ci/jenkins/scripts") { - sh "./build.sh -l" + dir ("ci/scripts") { withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -d /opt/milvus -j -u -c" + sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -d /opt/milvus -l -j -u -c" } } } diff --git a/ci/jenkins/step/coverage.groovy b/ci/jenkins/step/coverage.groovy index ff2e3e6fa2..75ac7b48b5 100644 --- a/ci/jenkins/step/coverage.groovy +++ b/ci/jenkins/step/coverage.groovy @@ -1,5 +1,5 @@ timeout(time: 30, unit: 'MINUTES') { - dir ("ci/jenkins/scripts") { + dir ("ci/scripts") { sh "./coverage.sh -o /opt/milvus -u root -p 123456 -t \$POD_IP" // Set some env variables so codecov detection script works correctly withCredentials([[$class: 'StringBinding', credentialsId: "${env.PIPELINE_NAME}-codecov-token", variable: 'CODECOV_TOKEN']]) { diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index a8bd339ff5..444537fe6b 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -18,7 +18,7 @@ INSTALL_PREFIX="/opt/milvus" BUILD_COVERAGE="OFF" USE_JFROG_CACHE="OFF" RUN_CPPLINT="OFF" -CPU_VERSION="ON" +GPU_VERSION="OFF" WITH_MKL="OFF" CUDA_COMPILER=/usr/local/cuda/bin/nvcc @@ -35,7 +35,7 @@ do CORE_BUILD_DIR=$OPTARG # CORE_BUILD_DIR ;; g) - CPU_VERSION="OFF"; + GPU_VERSION="ON"; ;; u) echo "Build and run unittest cases" ; @@ -84,13 +84,13 @@ if [[ ! -d ${CORE_BUILD_DIR} ]]; then mkdir ${CORE_BUILD_DIR} fi -pushd ${CORE_BUILD_DIR} +cd ${CORE_BUILD_DIR} CMAKE_CMD="cmake \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} -DCMAKE_BUILD_TYPE=${BUILD_TYPE} \ -DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \ --DMILVUS_CPU_VERSION=${CPU_VERSION} \ +-DMILVUS_GPU_VERSION=${GPU_VERSION} \ -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ -DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \ @@ -100,6 +100,34 @@ ${MILVUS_CORE_DIR}" echo ${CMAKE_CMD} ${CMAKE_CMD} + +if [[ ${RUN_CPPLINT} == "ON" ]]; then + # cpplint check + make lint + if [ $? -ne 0 ]; then + echo "ERROR! cpplint check failed" + exit 1 + fi + echo "cpplint check passed!" + + # clang-format check + make check-clang-format + if [ $? -ne 0 ]; then + echo "ERROR! clang-format check failed" + exit 1 + fi + echo "clang-format check passed!" + +# # clang-tidy check +# make check-clang-tidy +# if [ $? -ne 0 ]; then +# echo "ERROR! clang-tidy check failed" +# rm -f CMakeCache.txt +# exit 1 +# fi +# echo "clang-tidy check passed!" +fi + # compile and build make -j8 || exit 1 make install || exit 1 diff --git a/ci/jenkins/scripts/coverage.sh b/ci/scripts/coverage.sh similarity index 86% rename from ci/jenkins/scripts/coverage.sh rename to ci/scripts/coverage.sh index 5c9d010d46..17bc08ad00 100755 --- a/ci/jenkins/scripts/coverage.sh +++ b/ci/scripts/coverage.sh @@ -9,18 +9,22 @@ done SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" INSTALL_PREFIX="/opt/milvus" -CMAKE_BUILD_DIR="${SCRIPTS_DIR}/../../../core/cmake_build" +MILVUS_CORE_DIR="${SCRIPTS_DIR}/../../core" +CORE_BUILD_DIR="${MILVUS_CORE_DIR}/cmake_build" MYSQL_USER_NAME=root MYSQL_PASSWORD=123456 MYSQL_HOST='127.0.0.1' MYSQL_PORT='3306' -while getopts "o:u:p:t:h" arg +while getopts "o:b:u:p:t:h" arg do case $arg in o) INSTALL_PREFIX=$OPTARG ;; + b) + CORE_BUILD_DIR=$OPTARG # CORE_BUILD_DIR + ;; u) MYSQL_USER_NAME=$OPTARG ;; @@ -35,13 +39,14 @@ do parameter: -o: milvus install prefix(default: /opt/milvus) +-b: core code build directory -u: mysql account -p: mysql password -t: mysql host -h: help usage: -./coverage.sh -o \${INSTALL_PREFIX} -u \${MYSQL_USER} -p \${MYSQL_PASSWORD} -t \${MYSQL_HOST} [-h] +./coverage.sh -o \${INSTALL_PREFIX} -b \$${CORE_BUILD_DIR} -u \${MYSQL_USER} -p \${MYSQL_PASSWORD} -t \${MYSQL_HOST} [-h] " exit 0 ;; @@ -63,12 +68,14 @@ FILE_INFO_OUTPUT="output.info" FILE_INFO_OUTPUT_NEW="output_new.info" DIR_LCOV_OUTPUT="lcov_out" -DIR_GCNO="${CMAKE_BUILD_DIR}" +DIR_GCNO="${CORE_BUILD_DIR}" DIR_UNITTEST="${INSTALL_PREFIX}/unittest" +cd ${SCRIPTS_DIR} + # delete old code coverage info files -rm -rf lcov_out -rm -f FILE_INFO_BASE FILE_INFO_MILVUS FILE_INFO_OUTPUT FILE_INFO_OUTPUT_NEW +rm -rf ${DIR_LCOV_OUTPUT} +rm -f ${FILE_INFO_BASE} ${FILE_INFO_MILVUS} ${FILE_INFO_OUTPUT} ${FILE_INFO_OUTPUT_NEW} MYSQL_DB_NAME=milvus_`date +%s%N`