From 8644d341c452c3d9961c18bc19d830f4d1ba8574 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Tue, 17 Mar 2020 10:36:24 +0800 Subject: [PATCH] #1656 Add timeout for whole Jenkins job (#1676) * Increase memory limit in Jenkins CI Signed-off-by: quicksilver * Add pod tolerations to slave pods * Refactor ci/scripts/build.sh * Refactor ci/scripts/build.sh * Refactor CI * Refactor CI * Refactor CI * Refactor CI * Refactor CI * Refactor CI * Refactor CI * Add timeout for whole Jenkins job * cuda yum repository temporarily disabled --- ci/jenkins/Jenkinsfile | 49 +-------------------- ci/jenkins/step/build.groovy | 19 ++++---- ci/jenkins/step/coverage.groovy | 26 +++++------ ci/jenkins/step/internalCoverage.groovy | 6 --- ci/jenkins/step/package.groovy | 19 +++----- ci/jenkins/step/publishImages.groovy | 42 +++++++++--------- ci/jenkins/step/singleDevNightlyTest.groovy | 32 +++++++------- ci/jenkins/step/singleDevTest.groovy | 25 ++--------- ci/jenkins/step/unittest.groovy | 8 ++-- docker/deploy/gpu/centos7/Dockerfile | 4 +- 10 files changed, 72 insertions(+), 158 deletions(-) delete mode 100644 ci/jenkins/step/internalCoverage.groovy diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index b225c05bc1..98af7ee4d9 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -13,6 +13,7 @@ pipeline { options { timestamps() + timeout(time: 3, unit: 'HOURS') } parameters{ @@ -64,22 +65,6 @@ pipeline { stages { stage('Run Build') { - // when { - // allOf { - // anyOf { - // expression { isEmptyChangelog() == true } - // changeset 'ci/**' - // changeset 'core/**' - // changeset 'docker/**' - // changeset 'test/**' - // } - // not { - // anyOf { - // changeset '**/*.md' - // } - // } - // } - // } agent { kubernetes { label "${OS_NAME}-${BINARY_VERSION}-build-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}" @@ -140,22 +125,6 @@ pipeline { } stage('Publish docker images') { - // when { - // allOf { - // anyOf { - // expression { isEmptyChangelog() == true } - // changeset 'ci/**' - // changeset 'core/**' - // changeset 'docker/**' - // changeset 'test/**' - // } - // not { - // anyOf { - // changeset '**/*.md' - // } - // } - // } - // } agent { kubernetes { label "${OS_NAME}-${BINARY_VERSION}-publish-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}" @@ -178,22 +147,6 @@ pipeline { } stage('Deploy to Development') { - // when { - // allOf { - // anyOf { - // expression { isEmptyChangelog() == true } - // changeset 'ci/**' - // changeset 'core/**' - // changeset 'docker/**' - // changeset 'test/**' - // } - // not { - // anyOf { - // changeset '**/*.md' - // } - // } - // } - // } environment { FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-").replaceAll("_", "-") FORMAT_OS_NAME = "${OS_NAME}".replaceAll("\\.", "-").replaceAll("_", "-") diff --git a/ci/jenkins/step/build.groovy b/ci/jenkins/step/build.groovy index 3b91983bd5..ad8fdd4de6 100644 --- a/ci/jenkins/step/build.groovy +++ b/ci/jenkins/step/build.groovy @@ -1,14 +1,13 @@ -timeout(time: 75, unit: 'MINUTES') { - dir ("ci/scripts") { - withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - def checkResult = sh(script: "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache", returnStatus: true) +dir ("ci/scripts") { + withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { + def checkResult = sh(script: "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache", returnStatus: true) - if ("${BINARY_VERSION}" == "gpu") { - sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -i ${env.MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -g -u\"" - } else { - sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -i ${env.MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -u\"" - } - sh "./update_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache -u ${USERNAME} -p ${PASSWORD}" + if ("${BINARY_VERSION}" == "gpu") { + sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -i ${env.MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -g -u\"" + } else { + sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -i ${env.MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -u\"" } + sh "./update_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache -u ${USERNAME} -p ${PASSWORD}" } } + diff --git a/ci/jenkins/step/coverage.groovy b/ci/jenkins/step/coverage.groovy index c9844d4010..f0b257df3e 100644 --- a/ci/jenkins/step/coverage.groovy +++ b/ci/jenkins/step/coverage.groovy @@ -1,16 +1,14 @@ -timeout(time: 10, unit: 'MINUTES') { - dir ("ci/scripts") { - sh "./coverage.sh" - boolean isNightlyTest = currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0 ? true : false - String formatFlag = "${BINARY_VERSION}-version-${OS_NAME}-unittest".replaceAll("\\.", "_").replaceAll("-", "_") - if (isNightlyTest) { - withCredentials([[$class: 'StringBinding', credentialsId: "milvus-ci-codecov-token", variable: 'CODECOV_TOKEN']]) { - sh "curl -s https://codecov.io/bash | bash -s - -f output_new.info -n ${BINARY_VERSION}-version-${OS_NAME}-unittest -F nightly -F ${formatFlag} || echo \"Codecov did not collect coverage reports\"" - } - } else { - withCredentials([[$class: 'StringBinding', credentialsId: "milvus-ci-codecov-token", variable: 'CODECOV_TOKEN']]) { - sh "curl -s https://codecov.io/bash | bash -s - -f output_new.info -n ${BINARY_VERSION}-version-${OS_NAME}-unittest -F ${formatFlag} || echo \"Codecov did not collect coverage reports\"" - } - } +dir ("ci/scripts") { + sh "./coverage.sh" + boolean isNightlyTest = currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0 ? true : false + String formatFlag = "${BINARY_VERSION}-version-${OS_NAME}-unittest".replaceAll("\\.", "_").replaceAll("-", "_") + if (isNightlyTest) { + withCredentials([[$class: 'StringBinding', credentialsId: "milvus-ci-codecov-token", variable: 'CODECOV_TOKEN']]) { + sh "curl -s https://codecov.io/bash | bash -s - -f output_new.info -n ${BINARY_VERSION}-version-${OS_NAME}-unittest -F nightly -F ${formatFlag} || echo \"Codecov did not collect coverage reports\"" + } + } else { + withCredentials([[$class: 'StringBinding', credentialsId: "milvus-ci-codecov-token", variable: 'CODECOV_TOKEN']]) { + sh "curl -s https://codecov.io/bash | bash -s - -f output_new.info -n ${BINARY_VERSION}-version-${OS_NAME}-unittest -F ${formatFlag} || echo \"Codecov did not collect coverage reports\"" + } } } diff --git a/ci/jenkins/step/internalCoverage.groovy b/ci/jenkins/step/internalCoverage.groovy deleted file mode 100644 index 22ee1a1335..0000000000 --- a/ci/jenkins/step/internalCoverage.groovy +++ /dev/null @@ -1,6 +0,0 @@ -timeout(time: 30, unit: 'MINUTES') { - dir ("ci/scripts") { - sh "./coverage.sh -o ${env.MILVUS_INSTALL_PREFIX} -u root -p 123456 -t \$POD_IP" - } -} - diff --git a/ci/jenkins/step/package.groovy b/ci/jenkins/step/package.groovy index bbcc83e4c7..e65e79b42a 100644 --- a/ci/jenkins/step/package.groovy +++ b/ci/jenkins/step/package.groovy @@ -1,15 +1,8 @@ -timeout(time: 5, unit: 'MINUTES') { - // dir ("ci/jenkins/scripts") { - // sh "pip3 install -r requirements.txt" - // sh "./yaml_processor.py merge -f ${env.MILVUS_INSTALL_PREFIX}/conf/server_config.yaml -m ../yaml/update_server_config.yaml -i && rm ${env.MILVUS_INSTALL_PREFIX}/conf/server_config.yaml.bak" - // } - - sh "rm -rf ${env.MILVUS_INSTALL_PREFIX}/unittest" - sh "tar -zcvf ./${env.PROJECT_NAME}-${env.PACKAGE_VERSION}.tar.gz -C ${env.MILVUS_ROOT_PATH}/ ${env.PROJECT_NAME}" - withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { - def uploadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -T ./${env.PROJECT_NAME}-${env.PACKAGE_VERSION}.tar.gz ${params.JFROG_ARTFACTORY_URL}/milvus/package/${env.PROJECT_NAME}-${env.PACKAGE_VERSION}.tar.gz") - if (uploadStatus != 0) { - error("\" ${env.PROJECT_NAME}-${env.PACKAGE_VERSION}.tar.gz \" upload to \" ${params.JFROG_ARTFACTORY_URL}/milvus/package/${env.PROJECT_NAME}-${env.PACKAGE_VERSION}.tar.gz \" failed!") - } +sh "rm -rf ${env.MILVUS_INSTALL_PREFIX}/unittest" +sh "tar -zcvf ./${env.PROJECT_NAME}-${env.PACKAGE_VERSION}.tar.gz -C ${env.MILVUS_ROOT_PATH}/ ${env.PROJECT_NAME}" +withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { + def uploadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -T ./${env.PROJECT_NAME}-${env.PACKAGE_VERSION}.tar.gz ${params.JFROG_ARTFACTORY_URL}/milvus/package/${env.PROJECT_NAME}-${env.PACKAGE_VERSION}.tar.gz") + if (uploadStatus != 0) { + error("\" ${env.PROJECT_NAME}-${env.PACKAGE_VERSION}.tar.gz \" upload to \" ${params.JFROG_ARTFACTORY_URL}/milvus/package/${env.PROJECT_NAME}-${env.PACKAGE_VERSION}.tar.gz \" failed!") } } diff --git a/ci/jenkins/step/publishImages.groovy b/ci/jenkins/step/publishImages.groovy index e5f8d25f6c..8960ddd44f 100644 --- a/ci/jenkins/step/publishImages.groovy +++ b/ci/jenkins/step/publishImages.groovy @@ -1,33 +1,31 @@ -timeout(time: 15, unit: 'MINUTES') { - dir ("docker/deploy/${BINARY_VERSION}/${OS_NAME}") { - def binaryPackage = "${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz" +dir ("docker/deploy/${BINARY_VERSION}/${OS_NAME}") { + def binaryPackage = "${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz" - withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { - def downloadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -O ${params.JFROG_ARTFACTORY_URL}/milvus/package/${binaryPackage}") + withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { + def downloadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -O ${params.JFROG_ARTFACTORY_URL}/milvus/package/${binaryPackage}") - if (downloadStatus != 0) { - error("\" Download \" ${params.JFROG_ARTFACTORY_URL}/milvus/package/${binaryPackage} \" failed!") - } + if (downloadStatus != 0) { + error("\" Download \" ${params.JFROG_ARTFACTORY_URL}/milvus/package/${binaryPackage} \" failed!") } - sh "tar zxvf ${binaryPackage}" - def imageName = "${PROJECT_NAME}/engine:${DOCKER_VERSION}" + } + sh "tar zxvf ${binaryPackage}" + def imageName = "${PROJECT_NAME}/engine:${DOCKER_VERSION}" - try { - deleteImages("${imageName}", true) + try { + deleteImages("${imageName}", true) - def customImage = docker.build("${imageName}") + def customImage = docker.build("${imageName}") - deleteImages("${params.DOKCER_REGISTRY_URL}/${imageName}", true) + deleteImages("${params.DOKCER_REGISTRY_URL}/${imageName}", true) - docker.withRegistry("https://${params.DOKCER_REGISTRY_URL}", "${params.DOCKER_CREDENTIALS_ID}") { - customImage.push() - } - } catch (exc) { - throw exc - } finally { - deleteImages("${imageName}", true) - deleteImages("${params.DOKCER_REGISTRY_URL}/${imageName}", true) + docker.withRegistry("https://${params.DOKCER_REGISTRY_URL}", "${params.DOCKER_CREDENTIALS_ID}") { + customImage.push() } + } catch (exc) { + throw exc + } finally { + deleteImages("${imageName}", true) + deleteImages("${params.DOKCER_REGISTRY_URL}/${imageName}", true) } } diff --git a/ci/jenkins/step/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy index fc747f8fb5..38c762c8df 100644 --- a/ci/jenkins/step/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -1,21 +1,19 @@ -timeout(time: 90, unit: 'MINUTES') { - dir ("tests/milvus_python_test") { - // sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' - sh 'python3 -m pip install -r requirements.txt' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local" - } - // mysql database backend test - load "ci/jenkins/step/cleanupSingleDev.groovy" +dir ("tests/milvus_python_test") { + // sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' + sh 'python3 -m pip install -r requirements.txt' + sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local" +} +// mysql database backend test +load "ci/jenkins/step/cleanupSingleDev.groovy" - if (!fileExists('milvus-helm')) { - dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "master"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/master:refs/remotes/origin/master"]]]) - } - } +if (!fileExists('milvus-helm')) { dir ("milvus-helm") { - sh "helm install --wait --timeout 300s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." - } - dir ("tests/milvus_python_test") { - sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local" + checkout([$class: 'GitSCM', branches: [[name: "master"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/master:refs/remotes/origin/master"]]]) } } +dir ("milvus-helm") { + sh "helm install --wait --timeout 300s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." +} +dir ("tests/milvus_python_test") { + sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local" +} diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index 9f4882a56c..b5e33a4260 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -1,22 +1,5 @@ -timeout(time: 60, unit: 'MINUTES') { - dir ("tests/milvus_python_test") { - // sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' - sh 'python3 -m pip install -r requirements.txt' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local" - } - - // mysql database backend test - // load "ci/jenkins/step/cleanupSingleDev.groovy" - - // if (!fileExists('milvus-helm')) { - // dir ("milvus-helm") { - // checkout([$class: 'GitSCM', branches: [[name: "master"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/master:refs/remotes/origin/master"]]]) - // } - // } - // dir ("milvus-helm") { - // sh "helm install --wait --timeout 300s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." - // } - // dir ("tests/milvus_python_test") { - // sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local" - // } +dir ("tests/milvus_python_test") { + // sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' + sh 'python3 -m pip install -r requirements.txt' + sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local" } diff --git a/ci/jenkins/step/unittest.groovy b/ci/jenkins/step/unittest.groovy index f63e695664..195c9da41f 100644 --- a/ci/jenkins/step/unittest.groovy +++ b/ci/jenkins/step/unittest.groovy @@ -1,5 +1,3 @@ -timeout(time: 20, unit: 'MINUTES') { - dir ("ci/scripts") { - sh "./run_unittest.sh -i ${env.MILVUS_INSTALL_PREFIX} --mysql_user=root --mysql_password=123456 --mysql_host=\$POD_IP" - } -} \ No newline at end of file +dir ("ci/scripts") { + sh "./run_unittest.sh -i ${env.MILVUS_INSTALL_PREFIX} --mysql_user=root --mysql_password=123456 --mysql_host=\$POD_IP" +} diff --git a/docker/deploy/gpu/centos7/Dockerfile b/docker/deploy/gpu/centos7/Dockerfile index 61ad2d93d7..198c243949 100644 --- a/docker/deploy/gpu/centos7/Dockerfile +++ b/docker/deploy/gpu/centos7/Dockerfile @@ -13,8 +13,8 @@ FROM nvidia/cuda:10.1-devel-centos7 ENV NVIDIA_DRIVER_CAPABILITIES compute,utility -RUN yum install -y epel-release && \ - yum install -y libgomp libgfortran4 mysql-devel openblas-devel lapack-devel && \ +RUN yum --disablerepo=cuda install -y epel-release && \ + yum --disablerepo=cuda install -y libgomp libgfortran4 mysql-devel openblas-devel lapack-devel && \ rm -rf /var/cache/yum/* COPY ./milvus /var/lib/milvus