From 65a9f9e3e49bebc297044e6c6f279f4b4314a161 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 25 Nov 2019 19:54:08 +0800 Subject: [PATCH 1/7] add check_ccache.sh and update_ccache.sh --- ci/scripts/check_ccache.sh | 61 +++++++++++++++++++++++++++++++++++++ ci/scripts/update_ccache.sh | 55 +++++++++++++++++++++++++++++++++ 2 files changed, 116 insertions(+) create mode 100755 ci/scripts/check_ccache.sh create mode 100644 ci/scripts/update_ccache.sh diff --git a/ci/scripts/check_ccache.sh b/ci/scripts/check_ccache.sh new file mode 100755 index 0000000000..0eb277fdb9 --- /dev/null +++ b/ci/scripts/check_ccache.sh @@ -0,0 +1,61 @@ +#!/bin/bash + +OS_NAME="linux" +CODE_NAME=$(lsb_release -sc) +OS_MD5=$(lsb_release -s | md5sum | cut -d " " -f 1) +BRANCH_NAMES=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's=[a-zA-Z]*\/==g' | awk -F", " '{$1=""; print $0}') +ARTIFACTORY_URL="" +CCACHE_DIRECTORY="${HOME}/.ccache" + +while getopts "l:d:h" arg +do + case $arg in + l) + ARTIFACTORY_URL=$OPTARG + ;; + d) + CCACHE_DIRECTORY=$OPTARG + ;; + h) # help + echo " + +parameter: +-l: artifactory url +-d: ccache directory +-h: help + +usage: +./build.sh -l \${ARTIFACTORY_URL} -d \${CCACHE_DIRECTORY} [-h] + " + exit 0 + ;; + ?) + echo "ERROR! unknown argument" + exit 1 + ;; + esac +done + +if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then + echo "you have not input ARTIFACTORY_URL !" + exit 1 +fi + +for BRANCH_NAME in ${BRANCH_NAMES} +do + echo "fetching ${BRANCH_NAME}/ccache-${OS_NAME}-${CODE_NAME}-${OS_MD5}.tar.gz" + wget -q --method HEAD "${ARTIFACTORY_URL}/${BRANCH_NAME}/ccache-${OS_NAME}-${CODE_NAME}-${OS_MD5}.tar.gz" + if [[ $? == 0 ]];then + wget "${ARTIFACTORY_URL}/${BRANCH_NAME}/ccache-${OS_NAME}-${CODE_NAME}-${OS_MD5}.tar.gz" && \ + mkdir -p ${CCACHE_DIRECTORY} && \ + tar zxf ccache-${OS_NAME}-${CODE_NAME}-${OS_MD5}.tar.gz -C ${CCACHE_DIRECTORY} && \ + rm ccache-${OS_NAME}-${CODE_NAME}-${OS_MD5}.tar.gz + if [[ $? == 0 ]];then + echo "found cache" + exit 0 + fi + fi +done + +echo "could not download cache" && exit 1 + diff --git a/ci/scripts/update_ccache.sh b/ci/scripts/update_ccache.sh new file mode 100644 index 0000000000..5f167e4e73 --- /dev/null +++ b/ci/scripts/update_ccache.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +OS_NAME="linux" +CODE_NAME=$(lsb_release -sc) +OS_MD5=$(lsb_release -s | md5sum | cut -d " " -f 1) +BRANCH_NAMES=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's=[a-zA-Z]*\/==g' | awk -F", " '{$1=""; print $0}') +ARTIFACTORY_URL="" +ARTIFACTORY_USER="" +ARTIFACTORY_PASSWORD="" +CCACHE_DIRECTORY="${HOME}/.ccache" + +while getopts "l:u:p:d:h" arg +do + case $arg in + l) + ARTIFACTORY_URL=$OPTARG + ;; + u) + ARTIFACTORY_USER=$OPTARG + ;; + p) + ARTIFACTORY_PASSWORD=$OPTARG + ;; + d) + CCACHE_DIRECTORY=$OPTARG + ;; + h) # help + echo " + +parameter: +-l: artifactory url +-u: artifactory user +-p: artifactory password +-d: ccache directory +-h: help + +usage: +./build.sh -l \${ARTIFACTORY_URL} -u \${ARTIFACTORY_USER} -p \${ARTIFACTORY_PASSWORD} -d \${CCACHE_DIRECTORY} [-h] + " + exit 0 + ;; + ?) + echo "ERROR! unknown argument" + exit 1 + ;; + esac +done + +if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then + echo "you have not input ARTIFACTORY_URL !" + exit 1 +fi + +tar zcf ./ccache.tar.gz -C ${HOME}/.ccache . + From 75c25667b35d524041c5ceb01b2a794e626a472b Mon Sep 17 00:00:00 2001 From: ZhifengZhang-CN Date: Mon, 25 Nov 2019 22:46:52 +0800 Subject: [PATCH 2/7] update ci/scripts/update_ccache.sh --- ci/scripts/update_ccache.sh | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/ci/scripts/update_ccache.sh b/ci/scripts/update_ccache.sh index 5f167e4e73..c321aee64f 100644 --- a/ci/scripts/update_ccache.sh +++ b/ci/scripts/update_ccache.sh @@ -3,7 +3,7 @@ OS_NAME="linux" CODE_NAME=$(lsb_release -sc) OS_MD5=$(lsb_release -s | md5sum | cut -d " " -f 1) -BRANCH_NAMES=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's=[a-zA-Z]*\/==g' | awk -F", " '{$1=""; print $0}') +BRANCH_NAME=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's/.* \(.*\),.*/\1/' | sed 's=[a-zA-Z]*\/==g') ARTIFACTORY_URL="" ARTIFACTORY_USER="" ARTIFACTORY_PASSWORD="" @@ -51,5 +51,17 @@ if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then exit 1 fi -tar zcf ./ccache.tar.gz -C ${HOME}/.ccache . +PACKAGE_FILE="ccache-${OS_NAME}-${CODE_NAME}-${OS_MD5}.tar.gz" +REMOTE_PACKAGE_PATH="${ARTIFACTORY_URL}/${BRANCH_NAME}" +echo "Updating ccache package file: ${PACKAGE_FILE}" +tar zcf ./${PACKAGE_FILE} -C ${HOME}/.ccache . +echo "Uploading ccache package file ${PACKAGE_FILE} to ${REMOTE_PACKAGE_PATH}" +curl -u${ARTIFACTORY_USER}:${ARTIFACTORY_PASSWORD} -T ${PACKAGE_FILE} ${REMOTE_PACKAGE_PATH} +if [[ $? == 0 ]];then + echo "Uploading ccache package file success !" + exit 0 +else + echo "Uploading ccache package file fault !" + exit 1 +fi From 8528667e4a26ab48a3163e2f82261081024c1548 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Tue, 26 Nov 2019 11:05:44 +0800 Subject: [PATCH 3/7] add ccache in Dockerfile --- docker/build_env/cpu/ubuntu16.04/Dockerfile | 2 +- docker/build_env/cpu/ubuntu18.04/Dockerfile | 2 +- docker/build_env/gpu/ubuntu16.04/Dockerfile | 2 +- docker/build_env/gpu/ubuntu18.04/Dockerfile | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/docker/build_env/cpu/ubuntu16.04/Dockerfile b/docker/build_env/cpu/ubuntu16.04/Dockerfile index 45e2b53938..c44eaf44dc 100644 --- a/docker/build_env/cpu/ubuntu16.04/Dockerfile +++ b/docker/build_env/cpu/ubuntu16.04/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget ca-certifi sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \ wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ apt-get update && apt-get install -y --no-install-recommends \ - g++ git gfortran lsb-core \ + g++ git gfortran lsb-core ccache \ libboost-serialization-dev libboost-filesystem-dev libboost-system-dev libboost-regex-dev \ curl libtool automake libssl-dev pkg-config libcurl4-openssl-dev python3-pip \ clang-format-6.0 clang-tidy-6.0 \ diff --git a/docker/build_env/cpu/ubuntu18.04/Dockerfile b/docker/build_env/cpu/ubuntu18.04/Dockerfile index 7c76e2ec7a..8e4a90c819 100644 --- a/docker/build_env/cpu/ubuntu18.04/Dockerfile +++ b/docker/build_env/cpu/ubuntu18.04/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget ca-certifi sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \ wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ apt-get update && apt-get install -y --no-install-recommends \ - g++ git gfortran lsb-core \ + g++ git gfortran lsb-core ccache \ libboost-serialization-dev libboost-filesystem-dev libboost-system-dev libboost-regex-dev \ curl libtool automake libssl-dev pkg-config libcurl4-openssl-dev python3-pip \ clang-format-6.0 clang-tidy-6.0 \ diff --git a/docker/build_env/gpu/ubuntu16.04/Dockerfile b/docker/build_env/gpu/ubuntu16.04/Dockerfile index d35a7dccfd..55190f1f99 100644 --- a/docker/build_env/gpu/ubuntu16.04/Dockerfile +++ b/docker/build_env/gpu/ubuntu16.04/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget && \ apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \ apt-get update && apt-get install -y --no-install-recommends \ - git flex bison gfortran lsb-core \ + git flex bison gfortran lsb-core ccache \ curl libtool automake libboost1.58-all-dev libssl-dev pkg-config libcurl4-openssl-dev python3-pip \ clang-format-6.0 clang-tidy-6.0 \ lcov mysql-client libmysqlclient-dev intel-mkl-gnu-2019.5-281 intel-mkl-core-2019.5-281 && \ diff --git a/docker/build_env/gpu/ubuntu18.04/Dockerfile b/docker/build_env/gpu/ubuntu18.04/Dockerfile index 9f2f3f55ac..178551049c 100644 --- a/docker/build_env/gpu/ubuntu18.04/Dockerfile +++ b/docker/build_env/gpu/ubuntu18.04/Dockerfile @@ -8,7 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends wget && \ apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \ apt-get update && apt-get install -y --no-install-recommends \ - git flex bison gfortran lsb-core \ + git flex bison gfortran lsb-core ccache \ curl libtool automake libboost-all-dev libssl-dev pkg-config libcurl4-openssl-dev python3-pip \ clang-format-6.0 clang-tidy-6.0 \ lcov mysql-client libmysqlclient-dev intel-mkl-gnu-2019.5-281 intel-mkl-core-2019.5-281 && \ From 89555795a8f8e0c5ccfc85856bd2f9efad8626a1 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Tue, 26 Nov 2019 11:45:42 +0800 Subject: [PATCH 4/7] add ccache in build stage --- ci/jenkins/step/build.groovy | 11 +++++++---- ci/scripts/before-install.sh | 11 +++++++++++ ci/scripts/update_ccache.sh | 0 3 files changed, 18 insertions(+), 4 deletions(-) create mode 100755 ci/scripts/before-install.sh mode change 100644 => 100755 ci/scripts/update_ccache.sh diff --git a/ci/jenkins/step/build.groovy b/ci/jenkins/step/build.groovy index 6c1da64a82..dbca5aaba2 100644 --- a/ci/jenkins/step/build.groovy +++ b/ci/jenkins/step/build.groovy @@ -1,11 +1,14 @@ timeout(time: 60, unit: 'MINUTES') { dir ("ci/scripts") { withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { + sh "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache" if ("${env.BINRARY_VERSION}" == "gpu") { - sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -g -j -u -c" - } else { - sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -m -j -u -c" - } + sh "source ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -g -u -c" + } else { + sh "source ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -m -u -c" + } + + sh "./update_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache -u ${USERNAME} -p ${PASSWORD}" } } } diff --git a/ci/scripts/before-install.sh b/ci/scripts/before-install.sh new file mode 100755 index 0000000000..ad6cf17e00 --- /dev/null +++ b/ci/scripts/before-install.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +set -ex + +export CCACHE_COMPRESS=1 +export CCACHE_COMPRESSLEVEL=5 +export CCACHE_COMPILERCHECK=content +export PATH=/usr/lib/ccache/:$PATH +ccache --show-stats + +set +ex diff --git a/ci/scripts/update_ccache.sh b/ci/scripts/update_ccache.sh old mode 100644 new mode 100755 From bbb0d6ea23c9a3aaf989369d4c2567c27b492f95 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Tue, 26 Nov 2019 14:02:39 +0800 Subject: [PATCH 5/7] Update ci/jenkins/step/build.groovy --- ci/jenkins/step/build.groovy | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/jenkins/step/build.groovy b/ci/jenkins/step/build.groovy index dbca5aaba2..bff1073c8c 100644 --- a/ci/jenkins/step/build.groovy +++ b/ci/jenkins/step/build.groovy @@ -1,13 +1,12 @@ timeout(time: 60, unit: 'MINUTES') { dir ("ci/scripts") { withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache" + def checkResult = sh(script: "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache", returnStatus: true) if ("${env.BINRARY_VERSION}" == "gpu") { sh "source ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -g -u -c" } else { sh "source ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -m -u -c" } - sh "./update_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache -u ${USERNAME} -p ${PASSWORD}" } } From 64e95f590c0e975c42464c18d9836ec20cf5d6bb Mon Sep 17 00:00:00 2001 From: quicksilver Date: Tue, 26 Nov 2019 14:25:55 +0800 Subject: [PATCH 6/7] Update ci/scripts/build.groovy --- ci/jenkins/step/build.groovy | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ci/jenkins/step/build.groovy b/ci/jenkins/step/build.groovy index bff1073c8c..c18467f687 100644 --- a/ci/jenkins/step/build.groovy +++ b/ci/jenkins/step/build.groovy @@ -1,13 +1,11 @@ timeout(time: 60, unit: 'MINUTES') { dir ("ci/scripts") { withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - def checkResult = sh(script: "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache", returnStatus: true) if ("${env.BINRARY_VERSION}" == "gpu") { - sh "source ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -g -u -c" + sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -g -u -c" } else { - sh "source ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -m -u -c" + sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -m -u -c" } - sh "./update_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache -u ${USERNAME} -p ${PASSWORD}" } } } From c111c5361fe9f1bd961cc638915d406e4c70fbcf Mon Sep 17 00:00:00 2001 From: quicksilver Date: Tue, 26 Nov 2019 14:28:33 +0800 Subject: [PATCH 7/7] Update ci/scripts/build.groovy --- ci/jenkins/step/build.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/jenkins/step/build.groovy b/ci/jenkins/step/build.groovy index c18467f687..7bbc83c981 100644 --- a/ci/jenkins/step/build.groovy +++ b/ci/jenkins/step/build.groovy @@ -2,9 +2,9 @@ timeout(time: 60, unit: 'MINUTES') { dir ("ci/scripts") { withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { if ("${env.BINRARY_VERSION}" == "gpu") { - sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -g -u -c" + sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -g -j -u -c" } else { - sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -m -u -c" + sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -m -j -u -c" } } }