From 1b60e3bca6bd99c01f4825ba1d3c0275ca92ca1c Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sat, 14 Mar 2020 18:14:04 +0800 Subject: [PATCH] Refactor CI workflow (#1657) * Increase memory limit in Jenkins CI Signed-off-by: quicksilver * Add pod tolerations to slave pods * Refactor ci/scripts/build.sh * Refactor ci/scripts/build.sh * Refactor CI * Refactor CI * Refactor CI * Refactor CI * Refactor CI * Refactor CI * Refactor CI --- .github/workflows/core.yml | 4 +- ci/docker/centos-7-core.dockerfile | 6 + ci/docker/ubuntu-18.04-core.dockerfile | 6 + ci/jenkins/Jenkinsfile | 14 ++ ci/jenkins/step/build.groovy | 4 +- ci/jenkins/step/coverage.groovy | 4 +- ci/jenkins/step/unittest.groovy | 5 + ci/scripts/build.sh | 182 ++++++++++++++----------- ci/scripts/check_ccache.sh | 1 - ci/scripts/coverage.sh | 123 +++++------------ ci/scripts/run_unittest.sh | 121 ++++++++++++++++ docker-compose.yml | 16 +-- 12 files changed, 300 insertions(+), 186 deletions(-) create mode 100644 ci/jenkins/step/unittest.groovy create mode 100755 ci/scripts/run_unittest.sh diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index 8a3c7a7f65..9abea7d272 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -7,7 +7,7 @@ on: paths: - 'ci/**' - 'core/**' - - 'docker/**' + - '.github/workflows/core.yml' - '!**.md' - '!ci/jenkins/**' pull_request: @@ -15,7 +15,7 @@ on: paths: - 'ci/**' - 'core/**' - - 'docker/**' + - '.github/workflows/core.yml' - '!**.md' - '!ci/jenkins/**' diff --git a/ci/docker/centos-7-core.dockerfile b/ci/docker/centos-7-core.dockerfile index 263668af04..c70d21d13e 100644 --- a/ci/docker/centos-7-core.dockerfile +++ b/ci/docker/centos-7-core.dockerfile @@ -17,3 +17,9 @@ RUN echo "source scl_source enable devtoolset-7" >> /etc/profile.d/devtoolset-7. RUN echo "source scl_source enable llvm-toolset-7.0" >> /etc/profile.d/llvm-toolset-7.sh ENV CLANG_TOOLS_PATH="/opt/rh/llvm-toolset-7.0/root/usr/bin" + +# use login shell to activate environment un the RUN commands +SHELL [ "/bin/bash", "-c", "-l" ] + +# use login shell when running the container +ENTRYPOINT [ "/bin/bash", "-c", "-l" ] diff --git a/ci/docker/ubuntu-18.04-core.dockerfile b/ci/docker/ubuntu-18.04-core.dockerfile index 46a0901000..57891b8b92 100644 --- a/ci/docker/ubuntu-18.04-core.dockerfile +++ b/ci/docker/ubuntu-18.04-core.dockerfile @@ -25,3 +25,9 @@ RUN ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so \ /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so RUN sh -c 'echo export LD_LIBRARY_PATH=/opt/intel/compilers_and_libraries_2019.5.281/linux/mkl/lib/intel64:\$LD_LIBRARY_PATH > /etc/profile.d/mkl.sh' + +# use login shell to activate environment un the RUN commands +SHELL [ "/bin/bash", "-c", "-l" ] + +# use login shell when running the container +ENTRYPOINT [ "/bin/bash", "-c", "-l" ] diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 603f4cdbb8..b225c05bc1 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -99,6 +99,20 @@ pipeline { } } + stage('Unittest') { + steps { + container("milvus-${BINARY_VERSION}-build-env") { + script { + if ("${BINARY_VERSION}" == "gpu") { + load "${env.WORKSPACE}/ci/jenkins/step/unittest.groovy" + } else { + echo "Skip Unittest" + } + } + } + } + } + stage('Code Coverage') { steps { container("milvus-${BINARY_VERSION}-build-env") { diff --git a/ci/jenkins/step/build.groovy b/ci/jenkins/step/build.groovy index aaa106376f..3b91983bd5 100644 --- a/ci/jenkins/step/build.groovy +++ b/ci/jenkins/step/build.groovy @@ -4,9 +4,9 @@ timeout(time: 75, unit: 'MINUTES') { def checkResult = sh(script: "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache", returnStatus: true) if ("${BINARY_VERSION}" == "gpu") { - sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -g -u -c -i\"" + sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -i ${env.MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -g -u\"" } else { - sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -u -c -i\"" + sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -i ${env.MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -u\"" } sh "./update_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache -u ${USERNAME} -p ${PASSWORD}" } diff --git a/ci/jenkins/step/coverage.groovy b/ci/jenkins/step/coverage.groovy index 7155ca3051..c9844d4010 100644 --- a/ci/jenkins/step/coverage.groovy +++ b/ci/jenkins/step/coverage.groovy @@ -1,6 +1,6 @@ -timeout(time: 30, unit: 'MINUTES') { +timeout(time: 10, unit: 'MINUTES') { dir ("ci/scripts") { - sh "./coverage.sh -o ${env.MILVUS_INSTALL_PREFIX} -u root -p 123456 -t \$POD_IP" + sh "./coverage.sh" boolean isNightlyTest = currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0 ? true : false String formatFlag = "${BINARY_VERSION}-version-${OS_NAME}-unittest".replaceAll("\\.", "_").replaceAll("-", "_") if (isNightlyTest) { diff --git a/ci/jenkins/step/unittest.groovy b/ci/jenkins/step/unittest.groovy new file mode 100644 index 0000000000..f63e695664 --- /dev/null +++ b/ci/jenkins/step/unittest.groovy @@ -0,0 +1,5 @@ +timeout(time: 20, unit: 'MINUTES') { + dir ("ci/scripts") { + sh "./run_unittest.sh -i ${env.MILVUS_INSTALL_PREFIX} --mysql_user=root --mysql_password=123456 --mysql_host=\$POD_IP" + } +} \ No newline at end of file diff --git a/ci/scripts/build.sh b/ci/scripts/build.sh index 30ea8249d9..66ee6524bd 100755 --- a/ci/scripts/build.sh +++ b/ci/scripts/build.sh @@ -12,92 +12,106 @@ SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" MILVUS_CORE_DIR="${SCRIPTS_DIR}/../../core" CORE_BUILD_DIR="${MILVUS_CORE_DIR}/cmake_build" -BUILD_TYPE="Debug" -BUILD_UNITTEST="OFF" -INSTALL_PREFIX="/var/lib/milvus" -FAISS_ROOT="" -PRIVILEGES="OFF" -BUILD_COVERAGE="OFF" -RUN_CPPLINT="OFF" -GPU_VERSION="OFF" -WITH_MKL="OFF" -FIU_ENABLE="OFF" -CUDA_COMPILER=/usr/local/cuda/bin/nvcc -while getopts "o:t:b:f:pgulcmih" arg -do - case $arg in - o) - INSTALL_PREFIX=$OPTARG - ;; - t) - BUILD_TYPE=$OPTARG # BUILD_TYPE - ;; - b) - CORE_BUILD_DIR=$OPTARG # CORE_BUILD_DIR - ;; - f) - FAISS_ROOT=$OPTARG # FAISS ROOT PATH - ;; - p) - PRIVILEGES="ON" # ELEVATED PRIVILEGES - ;; - g) - GPU_VERSION="ON"; - ;; - u) - echo "Build and run unittest cases" ; - BUILD_UNITTEST="ON"; - ;; - l) - RUN_CPPLINT="ON" - ;; - c) - BUILD_COVERAGE="ON" - ;; - m) - WITH_MKL="ON" - ;; - i) - FIU_ENABLE="ON" - ;; - h) # help - echo " +HELP=" +Usage: + $0 [flags] [Arguments] -parameter: --o: install prefix(default: /var/lib/milvus) --t: build type(default: Debug) --b: core code build directory --f: faiss root path --p: install command with elevated privileges --g: gpu version --u: building unit test options(default: OFF) --l: run cpplint, clang-format and clang-tidy(default: OFF) --c: code coverage(default: OFF) --m: build with MKL(default: OFF) --i: build FIU_ENABLE(default: OFF) --h: help + clean Remove all existing build artifacts and configuration (start over) + -i [INSTALL_PREFIX] or --install_prefix=[INSTALL_PREFIX] + Install directory used by install. + -t [BUILD_TYPE] or --build_type=[BUILD_TYPE] + Build type (default: Release) + -j[N] or --jobs=[N] Allow N jobs at once; infinite jobs with no arg. + -l Run cpplint & check clang-format + -n No make and make install step + -g Building for the architecture of the GPU in the system + --with_mkl Build with MKL (default: OFF) + --with_fiu Build with FIU (default: OFF) + -c or --coverage Build Code Coverage + -u or --tests Build unittest case + -p or --privileges Install command with elevated privileges + -v or --verbose A level above ‘basic’; includes messages about which makefiles were parsed, prerequisites that did not need to be rebuilt + -h or --help Print help information -usage: -./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} -f \${FAISS_ROOT} [-p] [-g] [-u] [-l] [-c] [-m] [-i] [-h] - " - exit 0 - ;; - ?) - echo "ERROR! unknown argument" - exit 1 - ;; + +Use \"$0 --help\" for more information about a given command. +" + +ARGS=`getopt -o "i:t:j::lngcupvh" -l "install_prefix::,build_type::,jobs::,with_mkl,with_fiu,coverage,tests,privileges,help" -n "$0" -- "$@"` + +eval set -- "${ARGS}" + +while true ; do + case "$1" in + -i|--install_prefix) + # o has an optional argument. As we are in quoted mode, + # an empty parameter will be generated if its optional + # argument is not found. + case "$2" in + "") echo "Option install_prefix, no argument"; exit 1 ;; + *) INSTALL_PREFIX=$2 ; shift 2 ;; + esac ;; + -t|--build_type) + case "$2" in + "") echo "Option build_type, no argument"; exit 1 ;; + *) BUILD_TYPE=$2 ; shift 2 ;; + esac ;; + -j|--jobs) + case "$2" in + "") PARALLEL_LEVEL=""; shift 2 ;; + *) PARALLEL_LEVEL=$2 ; shift 2 ;; + esac ;; + -g) echo "Building for the architecture of the GPU in the system..." ; GPU_VERSION="ON" ; shift ;; + --with_mkl) echo "Build with MKL" ; WITH_MKL="ON" ; shift ;; + --with_fiu) echo "Build with FIU" ; FIU_ENABLE="ON" ; shift ;; + --coverage) echo "Build code coverage" ; BUILD_COVERAGE="ON" ; shift ;; + -u|--tests) echo "Build unittest cases" ; BUILD_UNITTEST="ON" ; shift ;; + -n) echo "No build and install step" ; COMPILE_BUILD="OFF" ; shift ;; + -l) RUN_CPPLINT="ON" ; shift ;; + -p|--privileges) PRIVILEGES="ON" ; shift ;; + -v|--verbose) VERBOSE="1" ; shift ;; + -h|--help) echo -e "${HELP}" ; exit 0 ;; + --) shift ; break ;; + *) echo "Internal error!" ; exit 1 ;; esac done -echo -e "===\n=== ccache statistics before build\n===" -ccache --show-stats +# Set defaults for vars modified by flags to this script +CUDA_COMPILER=/usr/local/cuda/bin/nvcc +INSTALL_PREFIX=${INSTALL_PREFIX:="/var/lib/milvus"} +VERBOSE=${VERBOSE:=""} +BUILD_TYPE=${BUILD_TYPE:="Release"} +BUILD_UNITTEST=${BUILD_UNITTEST:="OFF"} +BUILD_COVERAGE=${BUILD_COVERAGE:="OFF"} +COMPILE_BUILD=${COMPILE_BUILD:="ON"} +GPU_VERSION=${GPU_VERSION:="OFF"} +RUN_CPPLINT=${RUN_CPPLINT:="OFF"} +WITH_MKL=${WITH_MKL:="OFF"} +FIU_ENABLE=${FIU_ENABLE:="OFF"} +PRIVILEGES=${PRIVILEGES:="OFF"} +CLEANUP=${CLEANUP:="OFF"} +PARALLEL_LEVEL=${PARALLEL_LEVEL:="8"} + +for arg do +if [[ $arg == "clean" ]];then + echo "Remove all existing build artifacts and configuration..." + if [ -d ${CORE_BUILD_DIR} ]; then + find ${CORE_BUILD_DIR} -mindepth 1 -delete + rmdir ${CORE_BUILD_DIR} || true + fi + exit 0 +fi +done if [[ ! -d ${CORE_BUILD_DIR} ]]; then mkdir ${CORE_BUILD_DIR} fi -cd ${CORE_BUILD_DIR} +echo -e "===\n=== ccache statistics before build\n===" +ccache --show-stats + +pushd ${CORE_BUILD_DIR} CMAKE_CMD="cmake \ -DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX} @@ -106,7 +120,6 @@ CMAKE_CMD="cmake \ -DMILVUS_GPU_VERSION=${GPU_VERSION} \ -DBUILD_UNIT_TEST=${BUILD_UNITTEST} \ -DBUILD_COVERAGE=${BUILD_COVERAGE} \ --DFAISS_ROOT=${FAISS_ROOT} \ -DFAISS_WITH_MKL=${WITH_MKL} \ -DArrow_SOURCE=AUTO \ -DFAISS_SOURCE=AUTO \ @@ -115,7 +128,6 @@ ${MILVUS_CORE_DIR}" echo ${CMAKE_CMD} ${CMAKE_CMD} - if [[ ${RUN_CPPLINT} == "ON" ]]; then # cpplint check make lint @@ -143,11 +155,15 @@ if [[ ${RUN_CPPLINT} == "ON" ]]; then # echo "clang-tidy check passed!" fi -# compile and build -make -j6 || exit 1 +if [[ ${COMPILE_BUILD} == "ON" ]];then + # compile and build + make -j${PARALLEL_LEVEL} VERBOSE=${VERBOSE} || exit 1 -if [[ ${PRIVILEGES} == "ON" ]];then - sudo make install || exit 1 -else - make install || exit 1 + if [[ ${PRIVILEGES} == "ON" ]];then + sudo make install || exit 1 + else + make install || exit 1 + fi fi + +popd diff --git a/ci/scripts/check_ccache.sh b/ci/scripts/check_ccache.sh index e878ce9482..876ff83d64 100755 --- a/ci/scripts/check_ccache.sh +++ b/ci/scripts/check_ccache.sh @@ -94,4 +94,3 @@ do done echo "could not download cache" && exit 1 - diff --git a/ci/scripts/coverage.sh b/ci/scripts/coverage.sh index 2cd502870f..401794c96a 100755 --- a/ci/scripts/coverage.sh +++ b/ci/scripts/coverage.sh @@ -8,61 +8,46 @@ while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symli done SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" -INSTALL_PREFIX="/var/lib/milvus" -MILVUS_CORE_DIR="${SCRIPTS_DIR}/../../core" -CORE_BUILD_DIR="${MILVUS_CORE_DIR}/cmake_build" -MYSQL_USER_NAME=root -MYSQL_PASSWORD=123456 -MYSQL_HOST="127.0.0.1" -MYSQL_PORT="3306" -CODECOV_TOKEN="" +HELP=" +Usage: + $0 [flags] [Arguments] -while getopts "o:b:u:p:t:c:h" arg -do - case $arg in - o) - INSTALL_PREFIX=$OPTARG - ;; - b) - CORE_BUILD_DIR=$OPTARG # CORE_BUILD_DIR - ;; - u) - MYSQL_USER_NAME=$OPTARG - ;; - p) - MYSQL_PASSWORD=$OPTARG - ;; - t) - MYSQL_HOST=$OPTARG - ;; - c) - CODECOV_TOKEN=$OPTARG - ;; - h) # help - echo " + -b Core Code build directory + -c Codecov token + -h or --help Print help information -parameter: --o: milvus install prefix(default: /var/lib/milvus) --b: core code build directory --u: mysql account --p: mysql password --t: mysql host --c: codecov token --h: help -usage: -./coverage.sh -o \${INSTALL_PREFIX} -b \${CORE_BUILD_DIR} -u \${MYSQL_USER} -p \${MYSQL_PASSWORD} -t \${MYSQL_HOST} -c \${CODECOV_TOKEN} [-h] - " - exit 0 - ;; - ?) - echo "ERROR! unknown argument" - exit 1 - ;; +Use \"$0 --help\" for more information about a given command. +" + +ARGS=`getopt -o "b:c:h" -l "help" -n "$0" -- "$@"` + +eval set -- "${ARGS}" + +while true ; do + case "$1" in + -b) + # o has an optional argument. As we are in quoted mode, + # an empty parameter will be generated if its optional + # argument is not found. + case "$2" in + "") echo "Option CORE_BUILD_DIR, no argument"; exit 1 ;; + *) CORE_BUILD_DIR=$2 ; shift 2 ;; + esac ;; + -c) + case "$2" in + "") echo "Option CODECOV_TOKEN, no argument"; exit 1 ;; + *) CODECOV_TOKEN=$2 ; shift 2 ;; + esac ;; + -h|--help) echo -e "${HELP}" ; exit 0 ;; + --) shift ; break ;; + *) echo "Internal error!" ; exit 1 ;; esac done -export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${INSTALL_PREFIX}/lib +# Set defaults for vars modified by flags to this script +MILVUS_CORE_DIR="${SCRIPTS_DIR}/../../core" +CORE_BUILD_DIR=${CORE_BUILD_DIR:="${MILVUS_CORE_DIR}/cmake_build"} LCOV_CMD="lcov" # LCOV_GEN_CMD="genhtml" @@ -82,22 +67,6 @@ cd ${SCRIPTS_DIR} rm -rf ${DIR_LCOV_OUTPUT} rm -f ${FILE_INFO_BASE} ${FILE_INFO_MILVUS} ${FILE_INFO_OUTPUT} ${FILE_INFO_OUTPUT_NEW} -MYSQL_DB_NAME=milvus_`date +%s%N` - -function mysql_exc() -{ - cmd=$1 - mysql -h${MYSQL_HOST} -u${MYSQL_USER_NAME} -p${MYSQL_PASSWORD} -e "${cmd}" - if [ $? -ne 0 ]; then - echo "mysql $cmd run failed" - fi -} - -mysql_exc "CREATE DATABASE IF NOT EXISTS ${MYSQL_DB_NAME};" -mysql_exc "GRANT ALL PRIVILEGES ON ${MYSQL_DB_NAME}.* TO '${MYSQL_USER_NAME}'@'%';" -mysql_exc "FLUSH PRIVILEGES;" -mysql_exc "USE ${MYSQL_DB_NAME};" - # get baseline ${LCOV_CMD} -c -i -d ${DIR_GCNO} -o "${FILE_INFO_BASE}" if [ $? -ne 0 ]; then @@ -105,28 +74,6 @@ if [ $? -ne 0 ]; then exit -1 fi -for test in `ls ${DIR_UNITTEST}`; do - echo $test - case ${test} in - test_db) - # set run args for test_db - args="mysql://${MYSQL_USER_NAME}:${MYSQL_PASSWORD}@${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB_NAME}" - ;; - *test_*) - args="" - ;; - esac - # run unittest - ${DIR_UNITTEST}/${test} "${args}" - if [ $? -ne 0 ]; then - echo ${args} - echo ${DIR_UNITTEST}/${test} "run failed" - exit 1 - fi -done - -mysql_exc "DROP DATABASE IF EXISTS ${MYSQL_DB_NAME};" - # gen code coverage ${LCOV_CMD} -d ${DIR_GCNO} -o "${FILE_INFO_MILVUS}" -c # merge coverage @@ -151,7 +98,7 @@ if [ $? -ne 0 ]; then exit 2 fi -if [[ ! -z ${CODECOV_TOKEN} ]];then +if [[ -n ${CODECOV_TOKEN} ]];then export CODECOV_TOKEN="${CODECOV_TOKEN}" curl -s https://codecov.io/bash | bash -s - -f output_new.info || echo "Codecov did not collect coverage reports" fi diff --git a/ci/scripts/run_unittest.sh b/ci/scripts/run_unittest.sh new file mode 100755 index 0000000000..658b283a04 --- /dev/null +++ b/ci/scripts/run_unittest.sh @@ -0,0 +1,121 @@ +#!/bin/bash + +set -e + +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + +HELP=" +Usage: + $0 [flags] [Arguments] + + -i [INSTALL_PREFIX] or --install_prefix=[INSTALL_PREFIX] + Install directory used by install. + --mysql_user=[MYSQL_USER_NAME] MySQL User Name + --mysql_password=[MYSQL_PASSWORD] + MySQL Password + --mysql_host=[MYSQL_HOST] MySQL Host + -h or --help Print help information + + +Use \"$0 --help\" for more information about a given command. +" + +ARGS=`getopt -o "i:h" -l "install_prefix::,mysql_user::,mysql_password::,mysql_host::,help" -n "$0" -- "$@"` + +eval set -- "${ARGS}" + +while true ; do + case "$1" in + -i|--install_prefix) + # o has an optional argument. As we are in quoted mode, + # an empty parameter will be generated if its optional + # argument is not found. + case "$2" in + "") echo "Option install_prefix, no argument"; exit 1 ;; + *) INSTALL_PREFIX=$2 ; shift 2 ;; + esac ;; + --mysql_user) + case "$2" in + "") echo "Option mysql_user, no argument"; exit 1 ;; + *) MYSQL_USER_NAME=$2 ; shift 2 ;; + esac ;; + --mysql_password) + case "$2" in + "") echo "Option mysql_password, no argument"; exit 1 ;; + *) MYSQL_PASSWORD=$2 ; shift 2 ;; + esac ;; + --mysql_host) + case "$2" in + "") echo "Option mysql_host, no argument"; exit 1 ;; + *) MYSQL_HOST=$2 ; shift 2 ;; + esac ;; + -h|--help) echo -e "${HELP}" ; exit 0 ;; + --) shift ; break ;; + *) echo "Internal error!" ; exit 1 ;; + esac +done + +# Set defaults for vars modified by flags to this script +INSTALL_PREFIX=${INSTALL_PREFIX:="/var/lib/milvus"} +MYSQL_USER_NAME=${MYSQL_USER_NAME:="root"} +MYSQL_PASSWORD=${MYSQL_PASSWORD:="123456"} +MYSQL_HOST=${MYSQL_HOST:="127.0.0.1"} +MYSQL_PORT=${MYSQL_PORT:="3306"} +DIR_UNITTEST="${INSTALL_PREFIX}/unittest" + +if [ -d ${INSTALL_PREFIX}/lib ]; then + export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:${INSTALL_PREFIX}/lib +fi + +if [ ! -d ${DIR_UNITTEST} ]; then + echo "The unittest folder does not exist!" + exit 1 +fi + +pushd ${SCRIPTS_DIR} + +MYSQL_DB_NAME=milvus_`date +%s%N` + +function mysql_exc() +{ + cmd=$1 + mysql -h${MYSQL_HOST} -u${MYSQL_USER_NAME} -p${MYSQL_PASSWORD} -e "${cmd}" + if [ $? -ne 0 ]; then + echo "mysql $cmd run failed" + fi +} + +mysql_exc "CREATE DATABASE IF NOT EXISTS ${MYSQL_DB_NAME};" +mysql_exc "GRANT ALL PRIVILEGES ON ${MYSQL_DB_NAME}.* TO '${MYSQL_USER_NAME}'@'%';" +mysql_exc "FLUSH PRIVILEGES;" +mysql_exc "USE ${MYSQL_DB_NAME};" + +for test in `ls ${DIR_UNITTEST}`; do + echo $test + case ${test} in + test_db) + # set run args for test_db + args="mysql://${MYSQL_USER_NAME}:${MYSQL_PASSWORD}@${MYSQL_HOST}:${MYSQL_PORT}/${MYSQL_DB_NAME}" + ;; + *test_*) + args="" + ;; + esac + # run unittest + ${DIR_UNITTEST}/${test} "${args}" + if [ $? -ne 0 ]; then + echo ${args} + echo ${DIR_UNITTEST}/${test} "run failed" + exit 1 + fi +done + +mysql_exc "DROP DATABASE IF EXISTS ${MYSQL_DB_NAME};" + +popd \ No newline at end of file diff --git a/docker-compose.yml b/docker-compose.yml index 4e9b1ec0c7..72292fbb08 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -38,10 +38,10 @@ services: - ${ARCH}-ubuntu-${UBUNTU}-cache:/build:delegated networks: - milvus - command: &ubuntu-command > - /bin/bash -c " - /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c -i - /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql" + command: &ubuntu-command + ["/milvus/ci/scripts/build.sh -t Release -i ${MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -u && + /milvus/ci/scripts/run_unittest.sh -i ${MILVUS_INSTALL_PREFIX} --mysql_user=root --mysql_password=123456 --mysql_host=mysql && + /milvus/ci/scripts/coverage.sh"] centos-core: image: ${REPO}:${ARCH}-centos-${CENTOS}-core @@ -58,10 +58,10 @@ services: - ${ARCH}-centos-${CENTOS}-cache:/build:delegated networks: - milvus - command: ¢os-command > - /bin/bash --login -c " - /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c -i - /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql" + command: ¢os-command + ["/milvus/ci/scripts/build.sh -t Release -i ${MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -u && + /milvus/ci/scripts/run_unittest.sh -i ${MILVUS_INSTALL_PREFIX} --mysql_user=root --mysql_password=123456 --mysql_host=mysql && + /milvus/ci/scripts/coverage.sh"] networks: milvus: