diff --git a/ci/jenkins/step/build.groovy b/ci/jenkins/step/build.groovy index 6c1da64a82..7bbc83c981 100644 --- a/ci/jenkins/step/build.groovy +++ b/ci/jenkins/step/build.groovy @@ -2,10 +2,10 @@ timeout(time: 60, unit: 'MINUTES') { dir ("ci/scripts") { withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { if ("${env.BINRARY_VERSION}" == "gpu") { - sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -g -j -u -c" - } else { - sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -m -j -u -c" - } + sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -g -j -u -c" + } else { + sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -m -j -u -c" + } } } } diff --git a/ci/scripts/before-install.sh b/ci/scripts/before-install.sh new file mode 100755 index 0000000000..ad6cf17e00 --- /dev/null +++ b/ci/scripts/before-install.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -ex + +export CCACHE_COMPRESS=1 +export CCACHE_COMPRESSLEVEL=5 +export CCACHE_COMPILERCHECK=content +export PATH=/usr/lib/ccache/:$PATH +ccache --show-stats + +set +ex diff --git a/ci/scripts/check_ccache.sh b/ci/scripts/check_ccache.sh new file mode 100755 index 0000000000..0eb277fdb9 --- /dev/null +++ b/ci/scripts/check_ccache.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +OS_NAME="linux" +CODE_NAME=$(lsb_release -sc) +OS_MD5=$(lsb_release -s | md5sum | cut -d " " -f 1) +BRANCH_NAMES=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's=[a-zA-Z]*\/==g' | awk -F", " '{$1=""; print $0}') +ARTIFACTORY_URL="" +CCACHE_DIRECTORY="${HOME}/.ccache" + +while getopts "l:d:h" arg +do + case $arg in + l) + ARTIFACTORY_URL=$OPTARG + ;; + d) + CCACHE_DIRECTORY=$OPTARG + ;; + h) # help + echo " + +parameter: +-l: artifactory url +-d: ccache directory +-h: help + +usage: +./build.sh -l \${ARTIFACTORY_URL} -d \${CCACHE_DIRECTORY} [-h] + " + exit 0 + ;; + ?) + echo "ERROR! unknown argument" + exit 1 + ;; + esac +done + +if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then + echo "you have not input ARTIFACTORY_URL !" + exit 1 +fi + +for BRANCH_NAME in ${BRANCH_NAMES} +do + echo "fetching ${BRANCH_NAME}/ccache-${OS_NAME}-${CODE_NAME}-${OS_MD5}.tar.gz" + wget -q --method HEAD "${ARTIFACTORY_URL}/${BRANCH_NAME}/ccache-${OS_NAME}-${CODE_NAME}-${OS_MD5}.tar.gz" + if [[ $? == 0 ]];then + wget "${ARTIFACTORY_URL}/${BRANCH_NAME}/ccache-${OS_NAME}-${CODE_NAME}-${OS_MD5}.tar.gz" && \ + mkdir -p ${CCACHE_DIRECTORY} && \ + tar zxf ccache-${OS_NAME}-${CODE_NAME}-${OS_MD5}.tar.gz -C ${CCACHE_DIRECTORY} && \ + rm ccache-${OS_NAME}-${CODE_NAME}-${OS_MD5}.tar.gz + if [[ $? == 0 ]];then + echo "found cache" + exit 0 + fi + fi +done + +echo "could not download cache" && exit 1 + diff --git a/ci/scripts/update_ccache.sh b/ci/scripts/update_ccache.sh new file mode 100755 index 0000000000..c321aee64f --- /dev/null +++ b/ci/scripts/update_ccache.sh @@ -0,0 +1,67 @@ +#!/bin/bash + +OS_NAME="linux" +CODE_NAME=$(lsb_release -sc) +OS_MD5=$(lsb_release -s | md5sum | cut -d " " -f 1) +BRANCH_NAME=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's/.* \(.*\),.*/\1/' | sed 's=[a-zA-Z]*\/==g') +ARTIFACTORY_URL="" +ARTIFACTORY_USER="" +ARTIFACTORY_PASSWORD="" +CCACHE_DIRECTORY="${HOME}/.ccache" + +while getopts "l:u:p:d:h" arg +do + case $arg in + l) + ARTIFACTORY_URL=$OPTARG + ;; + u) + ARTIFACTORY_USER=$OPTARG + ;; + p) + ARTIFACTORY_PASSWORD=$OPTARG + ;; + d) + CCACHE_DIRECTORY=$OPTARG + ;; + h) # help + echo " + +parameter: +-l: artifactory url +-u: artifactory user +-p: artifactory password +-d: ccache directory +-h: help + +usage: +./build.sh -l \${ARTIFACTORY_URL} -u \${ARTIFACTORY_USER} -p \${ARTIFACTORY_PASSWORD} -d \${CCACHE_DIRECTORY} [-h] + " + exit 0 + ;; + ?) + echo "ERROR! unknown argument" + exit 1 + ;; + esac +done + +if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then + echo "you have not input ARTIFACTORY_URL !" + exit 1 +fi + +PACKAGE_FILE="ccache-${OS_NAME}-${CODE_NAME}-${OS_MD5}.tar.gz" +REMOTE_PACKAGE_PATH="${ARTIFACTORY_URL}/${BRANCH_NAME}" + +echo "Updating ccache package file: ${PACKAGE_FILE}" +tar zcf ./${PACKAGE_FILE} -C ${HOME}/.ccache . +echo "Uploading ccache package file ${PACKAGE_FILE} to ${REMOTE_PACKAGE_PATH}" +curl -u${ARTIFACTORY_USER}:${ARTIFACTORY_PASSWORD} -T ${PACKAGE_FILE} ${REMOTE_PACKAGE_PATH} +if [[ $? == 0 ]];then + echo "Uploading ccache package file success !" + exit 0 +else + echo "Uploading ccache package file fault !" + exit 1 +fi diff --git a/docker/build_env/cpu/ubuntu16.04/Dockerfile b/docker/build_env/cpu/ubuntu16.04/Dockerfile index 45e2b53938..c44eaf44dc 100644 --- a/docker/build_env/cpu/ubuntu16.04/Dockerfile +++ b/docker/build_env/cpu/ubuntu16.04/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget ca-certifi sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \ wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ apt-get update && apt-get install -y --no-install-recommends \ - g++ git gfortran lsb-core \ + g++ git gfortran lsb-core ccache \ libboost-serialization-dev libboost-filesystem-dev libboost-system-dev libboost-regex-dev \ curl libtool automake libssl-dev pkg-config libcurl4-openssl-dev python3-pip \ clang-format-6.0 clang-tidy-6.0 \ diff --git a/docker/build_env/cpu/ubuntu18.04/Dockerfile b/docker/build_env/cpu/ubuntu18.04/Dockerfile index 7c76e2ec7a..8e4a90c819 100644 --- a/docker/build_env/cpu/ubuntu18.04/Dockerfile +++ b/docker/build_env/cpu/ubuntu18.04/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget ca-certifi sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \ wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ apt-get update && apt-get install -y --no-install-recommends \ - g++ git gfortran lsb-core \ + g++ git gfortran lsb-core ccache \ libboost-serialization-dev libboost-filesystem-dev libboost-system-dev libboost-regex-dev \ curl libtool automake libssl-dev pkg-config libcurl4-openssl-dev python3-pip \ clang-format-6.0 clang-tidy-6.0 \ diff --git a/docker/build_env/gpu/ubuntu16.04/Dockerfile b/docker/build_env/gpu/ubuntu16.04/Dockerfile index d35a7dccfd..55190f1f99 100644 --- a/docker/build_env/gpu/ubuntu16.04/Dockerfile +++ b/docker/build_env/gpu/ubuntu16.04/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget && \ apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \ apt-get update && apt-get install -y --no-install-recommends \ - git flex bison gfortran lsb-core \ + git flex bison gfortran lsb-core ccache \ curl libtool automake libboost1.58-all-dev libssl-dev pkg-config libcurl4-openssl-dev python3-pip \ clang-format-6.0 clang-tidy-6.0 \ lcov mysql-client libmysqlclient-dev intel-mkl-gnu-2019.5-281 intel-mkl-core-2019.5-281 && \ diff --git a/docker/build_env/gpu/ubuntu18.04/Dockerfile b/docker/build_env/gpu/ubuntu18.04/Dockerfile index 9f2f3f55ac..178551049c 100644 --- a/docker/build_env/gpu/ubuntu18.04/Dockerfile +++ b/docker/build_env/gpu/ubuntu18.04/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget && \ apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \ apt-get update && apt-get install -y --no-install-recommends \ - git flex bison gfortran lsb-core \ + git flex bison gfortran lsb-core ccache \ curl libtool automake libboost-all-dev libssl-dev pkg-config libcurl4-openssl-dev python3-pip \ clang-format-6.0 clang-tidy-6.0 \ lcov mysql-client libmysqlclient-dev intel-mkl-gnu-2019.5-281 intel-mkl-core-2019.5-281 && \