diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index f6303e03c5..873c68ac02 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -48,7 +48,7 @@ pipeline { } axis { - name 'BINRARY_VERSION' + name 'BINARY_VERSION' values 'gpu', 'cpu' } } @@ -57,25 +57,25 @@ pipeline { stage("Milvus CI") { environment { PACKAGE_VERSION = VersionNumber([ - versionNumberString : '${SEMVER}-${BINRARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' + versionNumberString : '${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' ]); - DOCKER_VERSION = "${SEMVER}-${BINRARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}" + DOCKER_VERSION = "${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}" } stages { stage('Run Build') { agent { kubernetes { - label "${OS_NAME}-${BINRARY_VERSION}-build-${env.BUILD_NUMBER}" + label "${OS_NAME}-${BINARY_VERSION}-build-${env.BUILD_NUMBER}" defaultContainer 'jnlp' - yamlFile "ci/jenkins/pod/milvus-${BINRARY_VERSION}-version-${OS_NAME}-build-env-pod.yaml" + yamlFile "ci/jenkins/pod/milvus-${BINARY_VERSION}-version-${OS_NAME}-build-env-pod.yaml" } } stages { stage('Build') { steps { - container("milvus-${BINRARY_VERSION}-build-env") { + container("milvus-${BINARY_VERSION}-build-env") { script { load "${env.WORKSPACE}/ci/jenkins/step/build.groovy" } @@ -85,9 +85,9 @@ pipeline { stage('Code Coverage') { steps { - container("milvus-${BINRARY_VERSION}-build-env") { + container("milvus-${BINARY_VERSION}-build-env") { script { - if ("${BINRARY_VERSION}" == "gpu") { + if ("${BINARY_VERSION}" == "gpu") { load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" } else { echo "Skip Code Coverage" @@ -99,7 +99,7 @@ pipeline { stage('Upload Package') { steps { - container("milvus-${BINRARY_VERSION}-build-env") { + container("milvus-${BINARY_VERSION}-build-env") { script { load "${env.WORKSPACE}/ci/jenkins/step/package.groovy" } @@ -118,7 +118,7 @@ pipeline { stage('Publish docker images') { agent { kubernetes { - label "${OS_NAME}-${BINRARY_VERSION}-publish-${env.BUILD_NUMBER}" + label "${OS_NAME}-${BINARY_VERSION}-publish-${env.BUILD_NUMBER}" defaultContainer 'jnlp' yamlFile 'ci/jenkins/pod/docker-pod.yaml' } @@ -147,12 +147,12 @@ pipeline { environment { FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-") FORMAT_OS_NAME = "${OS_NAME}".replaceAll("\\.", "-") - HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${FORMAT_OS_NAME}-${BINRARY_VERSION}".toLowerCase() + HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${FORMAT_OS_NAME}-${BINARY_VERSION}".toLowerCase() } agent { kubernetes { - label "${OS_NAME}-${BINRARY_VERSION}-dev-test-${env.BUILD_NUMBER}" + label "${OS_NAME}-${BINARY_VERSION}-dev-test-${env.BUILD_NUMBER}" defaultContainer 'jnlp' yamlFile 'ci/jenkins/pod/testEnvironment.yaml' } diff --git a/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml b/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml index 6091a01eeb..f1628247b8 100644 --- a/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml @@ -16,6 +16,8 @@ spec: fieldPath: status.podIP - name: OS_NAME value: "centos7" + - name: BINARY_VERSION + value: "cpu" - name: BUILD_ENV_IMAGE_ID value: "225b4d9c26d67b70b476964b4dd6e216de4b464d7a973a8c0c7ed1313c4d81ad" command: diff --git a/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml b/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml index 4689da5db2..1c3fdc4bbf 100644 --- a/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml @@ -16,6 +16,8 @@ spec: fieldPath: status.podIP - name: OS_NAME value: "ubuntu18.04" + - name: BINARY_VERSION + value: "cpu" - name: BUILD_ENV_IMAGE_ID value: "23476391bec80c64f10d44a6370c73c71f011a6b95114b10ff82a60e771e11c7" command: diff --git a/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml b/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml index 34da55a93d..de3ce61630 100644 --- a/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml @@ -16,6 +16,8 @@ spec: fieldPath: status.podIP - name: OS_NAME value: "centos7" + - name: BINARY_VERSION + value: "gpu" - name: BUILD_ENV_IMAGE_ID value: "a5ec9914737ea4727d88ae36b4a73ca5d817f19438ba913cc1de6a1ee2ed2336" command: diff --git a/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml b/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml index 95b47c9111..8f941df63c 100644 --- a/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml @@ -16,6 +16,8 @@ spec: fieldPath: status.podIP - name: OS_NAME value: "ubuntu18.04" + - name: BINARY_VERSION + value: "gpu" - name: BUILD_ENV_IMAGE_ID value: "da9023b0f858f072672f86483a869aa87e90a5140864f89e5a012ec766d96dea" command: diff --git a/ci/jenkins/step/build.groovy b/ci/jenkins/step/build.groovy index d8b8f86f7b..2a12929575 100644 --- a/ci/jenkins/step/build.groovy +++ b/ci/jenkins/step/build.groovy @@ -3,7 +3,7 @@ timeout(time: 60, unit: 'MINUTES') { withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { def checkResult = sh(script: "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache", returnStatus: true) - if ("${BINRARY_VERSION}" == "gpu") { + if ("${BINARY_VERSION}" == "gpu") { sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -g -x -u -c\"" } else { sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -u -c\"" diff --git a/ci/jenkins/step/deploySingle2Dev.groovy b/ci/jenkins/step/deploySingle2Dev.groovy index e566c3da0c..e39dbcfb40 100644 --- a/ci/jenkins/step/deploySingle2Dev.groovy +++ b/ci/jenkins/step/deploySingle2Dev.groovy @@ -3,7 +3,7 @@ sh 'helm repo update' dir ('milvus-helm') { checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]]) dir ("milvus") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/sqlite_${BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/sqlite_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } diff --git a/ci/jenkins/step/publishImages.groovy b/ci/jenkins/step/publishImages.groovy index fe1c211abc..885fc5cf47 100644 --- a/ci/jenkins/step/publishImages.groovy +++ b/ci/jenkins/step/publishImages.groovy @@ -1,5 +1,5 @@ timeout(time: 15, unit: 'MINUTES') { - dir ("docker/deploy/${BINRARY_VERSION}/${OS_NAME}") { + dir ("docker/deploy/${BINARY_VERSION}/${OS_NAME}") { def binaryPackage = "${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz" withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { diff --git a/ci/jenkins/step/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy index 618c2457b3..0949041b01 100644 --- a/ci/jenkins/step/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -13,7 +13,7 @@ timeout(time: 90, unit: 'MINUTES') { } dir ("milvus-helm") { dir ("milvus") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } dir ("tests/milvus_python_test") { diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index 255f748996..279052eac1 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -14,7 +14,7 @@ timeout(time: 60, unit: 'MINUTES') { // } // dir ("milvus-helm") { // dir ("milvus") { - // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." // } // } // dir ("tests/milvus_python_test") { diff --git a/ci/scripts/check_ccache.sh b/ci/scripts/check_ccache.sh index c9c03ef149..98f95be7e2 100755 --- a/ci/scripts/check_ccache.sh +++ b/ci/scripts/check_ccache.sh @@ -1,6 +1,7 @@ #!/bin/bash OS_NAME="${OS_NAME}" +BINARY_VERSION="${BINARY_VERSION}" BUILD_ENV_DOCKER_IMAGE_ID="${BUILD_ENV_IMAGE_ID}" BRANCH_NAMES=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's=[a-zA-Z]*\/==g' | awk -F", " '{$1=""; print $0}') ARTIFACTORY_URL="" @@ -42,7 +43,7 @@ fi check_ccache() { BRANCH=$1 - PACKAGE_FILE="ccache-${OS_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" + PACKAGE_FILE="ccache-${OS_NAME}-${BINARY_VERSION}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" echo "fetching ${BRANCH}/${PACKAGE_FILE}" wget -q --spider "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" if [[ $? == 0 ]];then diff --git a/ci/scripts/update_ccache.sh b/ci/scripts/update_ccache.sh index ec52bd1156..605714fea5 100755 --- a/ci/scripts/update_ccache.sh +++ b/ci/scripts/update_ccache.sh @@ -1,6 +1,7 @@ #!/bin/bash OS_NAME="${OS_NAME}" +BINARY_VERSION="${BINARY_VERSION}" BUILD_ENV_DOCKER_IMAGE_ID="${BUILD_ENV_IMAGE_ID}" BRANCH_NAME=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's/.*, //' | sed 's=[a-zA-Z]*\/==g') ARTIFACTORY_URL="" @@ -50,7 +51,7 @@ if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then exit 1 fi -PACKAGE_FILE="ccache-${OS_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" +PACKAGE_FILE="ccache-${OS_NAME}-${BINARY_VERSION}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" REMOTE_PACKAGE_PATH="${ARTIFACTORY_URL}/${BRANCH_NAME}" ccache --show-stats