mirror of
https://gitee.com/milvus-io/milvus.git
synced 2026-02-02 01:06:41 +08:00
Update Jenkinsfile for improve workflow (#3228)
* Update Jenkinsfile Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update Jenkinsfile Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update Jenkinsfile Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update Jenkinsfile Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update Jenkinsfile Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update Jenkinsfile Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update Jenkinsfile Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update Jenkinsfile Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update Jenkinsfile Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update Jenkinsfile Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Test Dev deploy Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Merge singleDevTest.groovy Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Merge singleDevTest.groovy Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Modefied isTimeTriggeredBuild function Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update Jenkinsfile Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update Jenkinsfile Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update Jenkinsfile Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update Jenkinsfile Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update Jenkinsfile Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update Jenkinsfile Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update Jenkinsfile Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update Jenkinsfile Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update Jenkinsfile Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update Jenkinsfile Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update Jenkinsfile Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update Jenkinsfile Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update Jenkinsfile Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Update Jenkinsfile Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com>
This commit is contained in:
quicksilver
GitHub
parent
967c971a52
commit
4d3c7b7dfc
290
ci/jenkins/Jenkinsfile
vendored
290
ci/jenkins/Jenkinsfile
vendored
@ -5,16 +5,13 @@ String cron_string = BRANCH_NAME == "master" ? "50 22 * * * " : ""
|
||||
|
||||
pipeline {
|
||||
agent none
|
||||
|
||||
triggers {
|
||||
cron """${cron_timezone}
|
||||
${cron_string}"""
|
||||
}
|
||||
|
||||
options {
|
||||
timestamps()
|
||||
}
|
||||
|
||||
parameters{
|
||||
choice choices: ['Release', 'Debug'], description: 'Build Type', name: 'BUILD_TYPE'
|
||||
choice choices: ['False', 'True'], description: 'Is Manual Trigger Or Not', name: 'IS_MANUAL_TRIGGER_TYPE'
|
||||
@ -23,7 +20,6 @@ pipeline {
|
||||
string defaultValue: 'http://192.168.1.201/artifactory/milvus', description: 'JFROG ARTFACTORY URL', name: 'JFROG_ARTFACTORY_URL', trim: true
|
||||
string defaultValue: '1a527823-d2b7-44fd-834b-9844350baf14', description: 'JFROG CREDENTIALS ID', name: 'JFROG_CREDENTIALS_ID', trim: true
|
||||
}
|
||||
|
||||
environment {
|
||||
PROJECT_NAME = "milvus"
|
||||
MILVUS_ROOT_PATH="/var/lib"
|
||||
@ -33,15 +29,103 @@ pipeline {
|
||||
PIPELINE_NAME = "milvus-ci"
|
||||
HELM_BRANCH = "0.10.1"
|
||||
}
|
||||
|
||||
stages {
|
||||
stage ('Milvus CI') {
|
||||
stage ('Milvus Build and Unittest') {
|
||||
matrix {
|
||||
agent none
|
||||
axes {
|
||||
axis {
|
||||
name 'OS_NAME'
|
||||
values 'ubuntu18.04', 'centos7'
|
||||
values 'centos7'
|
||||
}
|
||||
axis {
|
||||
name 'CPU_ARCH'
|
||||
values 'amd64'
|
||||
}
|
||||
axis {
|
||||
name 'BINARY_VERSION'
|
||||
values 'gpu', 'cpu'
|
||||
}
|
||||
}
|
||||
environment {
|
||||
PACKAGE_VERSION = VersionNumber([
|
||||
versionNumberString : '${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}'
|
||||
]);
|
||||
}
|
||||
agent {
|
||||
kubernetes {
|
||||
label "${OS_NAME}-${BINARY_VERSION}-build-${SEMVER}-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"
|
||||
defaultContainer 'jnlp'
|
||||
customWorkspace '/home/jenkins/agent/workspace'
|
||||
yamlFile "ci/jenkins/pod/milvus-${BINARY_VERSION}-version-${OS_NAME}-build-env-pod.yaml"
|
||||
}
|
||||
}
|
||||
stages {
|
||||
stage('Build and Unittest') {
|
||||
steps {
|
||||
container("milvus-${BINARY_VERSION}-build-env") {
|
||||
script {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/build.groovy"
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/unittest.groovy"
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy"
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/package.groovy"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
stage ('Publish Docker Images') {
|
||||
matrix {
|
||||
axes {
|
||||
axis {
|
||||
name 'OS_NAME'
|
||||
values 'centos7'
|
||||
}
|
||||
axis {
|
||||
name 'CPU_ARCH'
|
||||
values 'amd64'
|
||||
}
|
||||
axis {
|
||||
name 'BINARY_VERSION'
|
||||
values 'gpu', 'cpu'
|
||||
}
|
||||
}
|
||||
environment {
|
||||
PACKAGE_VERSION = VersionNumber([
|
||||
versionNumberString : '${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}'
|
||||
]);
|
||||
SOURCE_REPO = "${params.DOKCER_REGISTRY_URL}/milvus/engine"
|
||||
TARGET_REPO = "${params.DOKCER_REGISTRY_URL}/milvus/engine"
|
||||
SOURCE_TAG = "${CHANGE_TARGET ? CHANGE_TARGET : SEMVER}-${BINARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}"
|
||||
TARGET_TAG = "${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}"
|
||||
}
|
||||
agent {
|
||||
kubernetes {
|
||||
label "${OS_NAME}-${BINARY_VERSION}-publish-${SEMVER}-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"
|
||||
defaultContainer 'jnlp'
|
||||
yamlFile 'ci/jenkins/pod/docker-pod.yaml'
|
||||
}
|
||||
}
|
||||
stages {
|
||||
stage('Publish') {
|
||||
steps {
|
||||
container('publish-images') {
|
||||
script {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
stage ('Dev Test') {
|
||||
matrix {
|
||||
axes {
|
||||
axis {
|
||||
name 'OS_NAME'
|
||||
values 'centos7'
|
||||
}
|
||||
|
||||
axis {
|
||||
@ -54,162 +138,43 @@ pipeline {
|
||||
values 'gpu', 'cpu'
|
||||
}
|
||||
}
|
||||
when {
|
||||
not {
|
||||
expression {return OS_NAME == "ubuntu18.04" && !isTimeTriggeredBuild()}
|
||||
}
|
||||
}
|
||||
|
||||
environment {
|
||||
PACKAGE_VERSION = VersionNumber([
|
||||
versionNumberString : '${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}'
|
||||
]);
|
||||
DOCKER_VERSION = "${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}"
|
||||
CHANGE_DOCKER_VERSION = "${CHANGE_TARGET}-${BINARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}"
|
||||
CACHA_DOCKER_VERSION = "${CHANGE_TARGET ? CHANGE_DOCKER_VERSION : DOCKER_VERSION}"
|
||||
FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-").replaceAll("_", "-")
|
||||
FORMAT_OS_NAME = "${OS_NAME}".replaceAll("\\.", "-").replaceAll("_", "-")
|
||||
HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${FORMAT_OS_NAME}-${BINARY_VERSION}".toLowerCase()
|
||||
SHARDS_HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-shards-${FORMAT_OS_NAME}-${BINARY_VERSION}".toLowerCase()
|
||||
DEV_TEST_ARTIFACTS = "_artifacts/${FORMAT_OS_NAME}/${BINARY_VERSION}"
|
||||
}
|
||||
agent {
|
||||
kubernetes {
|
||||
label "${OS_NAME}-${BINARY_VERSION}-dev-test-${SEMVER}-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"
|
||||
defaultContainer 'jnlp'
|
||||
yamlFile 'ci/jenkins/pod/testEnvironment.yaml'
|
||||
}
|
||||
}
|
||||
|
||||
stages {
|
||||
stage("Milvus Build and Unittest") {
|
||||
agent {
|
||||
kubernetes {
|
||||
label "${OS_NAME}-${BINARY_VERSION}-build-${SEMVER}-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"
|
||||
defaultContainer 'jnlp'
|
||||
customWorkspace '/home/jenkins/agent/workspace'
|
||||
yamlFile "ci/jenkins/pod/milvus-${BINARY_VERSION}-version-${OS_NAME}-build-env-pod.yaml"
|
||||
}
|
||||
}
|
||||
|
||||
stages {
|
||||
stage('Build') {
|
||||
steps {
|
||||
container("milvus-${BINARY_VERSION}-build-env") {
|
||||
script {
|
||||
try{
|
||||
boolean isNightlyTest = isTimeTriggeredBuild()
|
||||
if (isNightlyTest || "${params.IS_MANUAL_TRIGGER_TYPE}" == "True") {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/nightlyBuild.groovy"
|
||||
} else {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/build.groovy"
|
||||
}
|
||||
} catch (Exception e) {
|
||||
containerLog "milvus-${BINARY_VERSION}-build-env"
|
||||
throw e
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Unittest') {
|
||||
steps {
|
||||
container("milvus-${BINARY_VERSION}-build-env") {
|
||||
script {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/unittest.groovy"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Code Coverage') {
|
||||
steps {
|
||||
container("milvus-${BINARY_VERSION}-build-env") {
|
||||
script {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Upload Package') {
|
||||
steps {
|
||||
container("milvus-${BINARY_VERSION}-build-env") {
|
||||
script {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/package.groovy"
|
||||
}
|
||||
stage('Test') {
|
||||
steps {
|
||||
container('milvus-test-env') {
|
||||
script {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy"
|
||||
if (isTimeTriggeredBuild() || "${params.IS_MANUAL_TRIGGER_TYPE}" == "True") {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/shardsDevTest.groovy"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Publish Docker images') {
|
||||
environment {
|
||||
SOURCE_REPO = "${params.DOKCER_REGISTRY_URL}/milvus/engine"
|
||||
TARGET_REPO = "${params.DOKCER_REGISTRY_URL}/milvus/engine"
|
||||
SOURCE_TAG = "${CACHA_DOCKER_VERSION}"
|
||||
TARGET_TAG = "${DOCKER_VERSION}"
|
||||
}
|
||||
|
||||
agent {
|
||||
kubernetes {
|
||||
label "${OS_NAME}-${BINARY_VERSION}-publish-${SEMVER}-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"
|
||||
defaultContainer 'jnlp'
|
||||
yamlFile 'ci/jenkins/pod/docker-pod.yaml'
|
||||
}
|
||||
}
|
||||
|
||||
stages {
|
||||
stage('Publish') {
|
||||
steps {
|
||||
container('publish-images') {
|
||||
script {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Deploy to Development') {
|
||||
environment {
|
||||
FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-").replaceAll("_", "-")
|
||||
FORMAT_OS_NAME = "${OS_NAME}".replaceAll("\\.", "-").replaceAll("_", "-")
|
||||
HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${FORMAT_OS_NAME}-${BINARY_VERSION}".toLowerCase()
|
||||
SHARDS_HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-shards-${FORMAT_OS_NAME}-${BINARY_VERSION}".toLowerCase()
|
||||
DEV_TEST_ARTIFACTS = "_artifacts/${FORMAT_OS_NAME}/${BINARY_VERSION}"
|
||||
}
|
||||
|
||||
agent {
|
||||
kubernetes {
|
||||
label "${OS_NAME}-${BINARY_VERSION}-dev-test-${SEMVER}-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"
|
||||
defaultContainer 'jnlp'
|
||||
yamlFile 'ci/jenkins/pod/testEnvironment.yaml'
|
||||
}
|
||||
}
|
||||
|
||||
stages {
|
||||
stage('Dev Test') {
|
||||
steps {
|
||||
container('milvus-test-env') {
|
||||
script {
|
||||
sh "mkdir -p ${env.DEV_TEST_ARTIFACTS}"
|
||||
|
||||
boolean isNightlyTest = isTimeTriggeredBuild()
|
||||
if (isNightlyTest || "${params.IS_MANUAL_TRIGGER_TYPE}" == "True") {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy"
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/shardsDevNightlyTest.groovy"
|
||||
} else {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
post {
|
||||
cleanup {
|
||||
container('milvus-test-env') {
|
||||
script {
|
||||
archiveArtifacts artifacts: "${env.DEV_TEST_ARTIFACTS}/**", allowEmptyArchive: true
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy"
|
||||
boolean isNightlyTest = isTimeTriggeredBuild()
|
||||
if (isNightlyTest || "${params.IS_MANUAL_TRIGGER_TYPE}" == "True") {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/cleanupShardsDev.groovy"
|
||||
}
|
||||
}
|
||||
}
|
||||
post {
|
||||
cleanup {
|
||||
container('milvus-test-env') {
|
||||
script {
|
||||
archiveArtifacts artifacts: "${env.DEV_TEST_ARTIFACTS}/**", allowEmptyArchive: true
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDevTest.groovy"
|
||||
if (isTimeTriggeredBuild() || "${params.IS_MANUAL_TRIGGER_TYPE}" == "True") {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/cleanupShardsDevTest.groovy"
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -221,8 +186,7 @@ pipeline {
|
||||
post {
|
||||
unsuccessful {
|
||||
script {
|
||||
boolean isNightlyTest = isTimeTriggeredBuild()
|
||||
if (isNightlyTest) {
|
||||
if (isTimeTriggeredBuild()) {
|
||||
// Send an email only if the build status has changed from green/unstable to red
|
||||
emailext subject: '$DEFAULT_SUBJECT',
|
||||
body: '$DEFAULT_CONTENT',
|
||||
@ -238,16 +202,6 @@ pipeline {
|
||||
}
|
||||
}
|
||||
|
||||
boolean isEmptyChangelog() {
|
||||
if (currentBuild.changeSets.size() == 0) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
boolean isTimeTriggeredBuild() {
|
||||
if (currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0) {
|
||||
return true
|
||||
}
|
||||
return false
|
||||
return (currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0) ? true : false;
|
||||
}
|
||||
|
||||
@ -1,10 +1,16 @@
|
||||
timeout(time: 120, unit: 'MINUTES') {
|
||||
dir ("ci/scripts") {
|
||||
withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
|
||||
def checkResult = sh(script: "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache", returnStatus: true)
|
||||
if (!isTimeTriggeredBuild()) {
|
||||
sh "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache || echo \"ccache files not found!\""
|
||||
}
|
||||
|
||||
if ("${BINARY_VERSION}" == "gpu") {
|
||||
sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -j4 -i ${env.MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -g -u -s '-gencode=arch=compute_61,code=sm_61;-gencode=arch=compute_75,code=sm_75' \""
|
||||
if (isTimeTriggeredBuild()) {
|
||||
sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -j4 -i ${env.MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -g -u -s '-gencode=arch=compute_61,code=sm_61;-gencode=arch=compute_75,code=sm_75' \""
|
||||
} else {
|
||||
sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -j4 -i ${env.MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -g -u\""
|
||||
}
|
||||
} else {
|
||||
sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -j4 -i ${env.MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -u\""
|
||||
}
|
||||
@ -12,3 +18,7 @@ timeout(time: 120, unit: 'MINUTES') {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
boolean isTimeTriggeredBuild() {
|
||||
return (currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0) ? true : false;
|
||||
}
|
||||
|
||||
@ -1,12 +0,0 @@
|
||||
try {
|
||||
def helmResult = sh script: "helm status -n milvus ${env.SHARDS_HELM_RELEASE_NAME}", returnStatus: true
|
||||
if (!helmResult) {
|
||||
sh "helm uninstall -n milvus ${env.SHARDS_HELM_RELEASE_NAME}"
|
||||
}
|
||||
} catch (exc) {
|
||||
def helmResult = sh script: "helm status -n milvus ${env.SHARDS_HELM_RELEASE_NAME}", returnStatus: true
|
||||
if (!helmResult) {
|
||||
sh "helm uninstall -n milvus ${env.SHARDS_HELM_RELEASE_NAME}"
|
||||
}
|
||||
throw exc
|
||||
}
|
||||
6
ci/jenkins/step/cleanupShardsDevTest.groovy
Normal file
6
ci/jenkins/step/cleanupShardsDevTest.groovy
Normal file
@ -0,0 +1,6 @@
|
||||
retry(3) {
|
||||
def helmResult = sh script: "helm status -n milvus ${env.SHARDS_HELM_RELEASE_NAME}", returnStatus: true
|
||||
if (!helmResult) {
|
||||
sh "helm uninstall -n milvus ${env.SHARDS_HELM_RELEASE_NAME}"
|
||||
}
|
||||
}
|
||||
@ -1,12 +0,0 @@
|
||||
try {
|
||||
def helmResult = sh script: "helm status -n milvus ${env.HELM_RELEASE_NAME}", returnStatus: true
|
||||
if (!helmResult) {
|
||||
sh "helm uninstall -n milvus ${env.HELM_RELEASE_NAME}"
|
||||
}
|
||||
} catch (exc) {
|
||||
def helmResult = sh script: "helm status -n milvus ${env.HELM_RELEASE_NAME}", returnStatus: true
|
||||
if (!helmResult) {
|
||||
sh "helm uninstall -n milvus ${env.HELM_RELEASE_NAME}"
|
||||
}
|
||||
throw exc
|
||||
}
|
||||
6
ci/jenkins/step/cleanupSingleDevTest.groovy
Normal file
6
ci/jenkins/step/cleanupSingleDevTest.groovy
Normal file
@ -0,0 +1,6 @@
|
||||
retry(3) {
|
||||
def helmResult = sh script: "helm status -n milvus ${env.HELM_RELEASE_NAME}", returnStatus: true
|
||||
if (!helmResult) {
|
||||
sh "helm uninstall -n milvus ${env.HELM_RELEASE_NAME}"
|
||||
}
|
||||
}
|
||||
@ -1,9 +1,8 @@
|
||||
timeout(time: 30, unit: 'MINUTES') {
|
||||
dir ("ci/scripts") {
|
||||
sh ". ./before-install.sh && ./coverage.sh"
|
||||
boolean isNightlyTest = currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0 ? true : false
|
||||
String formatFlag = "${BINARY_VERSION}-version-${OS_NAME}-unittest".replaceAll("\\.", "_").replaceAll("-", "_")
|
||||
if (isNightlyTest) {
|
||||
if (isTimeTriggeredBuild()) {
|
||||
withCredentials([[$class: 'StringBinding', credentialsId: "milvus-ci-codecov-token", variable: 'CODECOV_TOKEN']]) {
|
||||
sh "curl -s https://codecov.io/bash | bash -s - -f output_new.info -U \"--proxy http://proxy.zilliz.tech:1088\" -A \"--proxy http://proxy.zilliz.tech:1088\" -n ${BINARY_VERSION}-version-${OS_NAME}-unittest -F nightly -F ${formatFlag} || echo \"Codecov did not collect coverage reports\""
|
||||
}
|
||||
@ -14,3 +13,7 @@ timeout(time: 30, unit: 'MINUTES') {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
boolean isTimeTriggeredBuild() {
|
||||
return (currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0) ? true : false;
|
||||
}
|
||||
|
||||
@ -1,12 +0,0 @@
|
||||
timeout(time: 120, unit: 'MINUTES') {
|
||||
dir ("ci/scripts") {
|
||||
withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) {
|
||||
if ("${BINARY_VERSION}" == "gpu") {
|
||||
sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -j4 -i ${env.MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -g -u\""
|
||||
} else {
|
||||
sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -j4 -i ${env.MILVUS_INSTALL_PREFIX} --with_fiu --coverage -l -u\""
|
||||
}
|
||||
sh "./update_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache -u ${USERNAME} -p ${PASSWORD}"
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,4 +1,6 @@
|
||||
timeout(time: 180, unit: 'MINUTES') {
|
||||
sh "mkdir -p ${env.DEV_TEST_ARTIFACTS}"
|
||||
|
||||
dir ('milvus-helm') {
|
||||
sh 'helm version'
|
||||
sh 'helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
|
||||
@ -1,58 +0,0 @@
|
||||
timeout(time: 180, unit: 'MINUTES') {
|
||||
dir ('milvus-helm') {
|
||||
sh 'helm version'
|
||||
sh 'helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
|
||||
sh 'helm repo update'
|
||||
checkout([$class: 'GitSCM', branches: [[name: "${env.HELM_BRANCH}"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${env.HELM_BRANCH}:refs/remotes/origin/${env.HELM_BRANCH}"]]])
|
||||
// sh 'helm dep update'
|
||||
|
||||
retry(3) {
|
||||
try {
|
||||
dir ('charts/milvus') {
|
||||
sh "helm install --wait --timeout 300s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP --set image.resources.requests.memory=8Gi --set image.resources.requests.cpu=2.0 --set image.resources.limits.memory=12Gi --set image.resources.limits.cpu=4.0 -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ."
|
||||
}
|
||||
} catch (exc) {
|
||||
def helmStatusCMD = "helm get manifest --namespace milvus ${env.HELM_RELEASE_NAME} | kubectl describe -n milvus -f - && \
|
||||
kubectl logs --namespace milvus -l \"app.kubernetes.io/name=milvus,app.kubernetes.io/instance=${env.HELM_RELEASE_NAME}\" -c milvus && \
|
||||
helm status -n milvus ${env.HELM_RELEASE_NAME}"
|
||||
sh script: helmStatusCMD, returnStatus: true
|
||||
sh script: "helm uninstall -n milvus ${env.HELM_RELEASE_NAME} && sleep 1m", returnStatus: true
|
||||
throw exc
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dir ("tests/milvus_python_test") {
|
||||
// sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
|
||||
sh 'python3 -m pip install -r requirements.txt'
|
||||
sh "pytest . --level=2 --alluredir=\"test_out/dev/single/mysql\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local >> ${WORKSPACE}/${env.DEV_TEST_ARTIFACTS}/milvus_${BINARY_VERSION}_mysql_dev_test.log"
|
||||
}
|
||||
// sqlite database backend test
|
||||
load "ci/jenkins/step/cleanupSingleDev.groovy"
|
||||
|
||||
if (!fileExists('milvus-helm/charts/milvus')) {
|
||||
dir ("milvus-helm") {
|
||||
checkout([$class: 'GitSCM', branches: [[name:"${env.HELM_BRANCH}"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${env.HELM_BRANCH}:refs/remotes/origin/${env.HELM_BRANCH}"]]])
|
||||
}
|
||||
}
|
||||
retry(3) {
|
||||
try {
|
||||
dir ("milvus-helm/charts/milvus") {
|
||||
sh "helm install --wait --timeout 300s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP --set image.resources.requests.memory=8Gi --set image.resources.requests.cpu=2.0 --set image.resources.limits.memory=12Gi --set image.resources.limits.cpu=4.0 -f ci/db_backend/sqlite_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ."
|
||||
}
|
||||
} catch (exc) {
|
||||
def helmStatusCMD = "helm get manifest --namespace milvus ${env.HELM_RELEASE_NAME} | kubectl describe -n milvus -f - && \
|
||||
kubectl logs --namespace milvus -l \"app=milvus,release=${env.HELM_RELEASE_NAME}\" -c milvus && \
|
||||
helm status -n milvus ${env.HELM_RELEASE_NAME}"
|
||||
def helmResult = sh script: helmStatusCMD, returnStatus: true
|
||||
if (!helmResult) {
|
||||
sh "helm uninstall -n milvus ${env.HELM_RELEASE_NAME} && sleep 1m"
|
||||
}
|
||||
throw exc
|
||||
}
|
||||
}
|
||||
dir ("tests/milvus_python_test") {
|
||||
sh "pytest . --level=2 --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local >> ${WORKSPACE}/${env.DEV_TEST_ARTIFACTS}/milvus_${BINARY_VERSION}_sqlite_dev_test.log"
|
||||
sh "pytest . --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local --port=19121 --handler=HTTP >> ${WORKSPACE}/${env.DEV_TEST_ARTIFACTS}/milvus_${BINARY_VERSION}_sqlite_http_dev_test.log"
|
||||
}
|
||||
}
|
||||
@ -1,10 +1,12 @@
|
||||
timeout(time: 120, unit: 'MINUTES') {
|
||||
sh "mkdir -p ${env.DEV_TEST_ARTIFACTS}"
|
||||
|
||||
dir ('milvus-helm') {
|
||||
sh 'helm version'
|
||||
sh 'helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
|
||||
sh 'helm repo update'
|
||||
sh "helm version && \
|
||||
helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts && \
|
||||
helm repo update"
|
||||
|
||||
checkout([$class: 'GitSCM', branches: [[name: "${env.HELM_BRANCH}"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${env.HELM_BRANCH}:refs/remotes/origin/${env.HELM_BRANCH}"]]])
|
||||
// sh 'helm dep update'
|
||||
|
||||
retry(3) {
|
||||
try {
|
||||
@ -21,9 +23,43 @@ timeout(time: 120, unit: 'MINUTES') {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
dir ("tests/milvus_python_test") {
|
||||
sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
|
||||
sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local --service ${env.HELM_RELEASE_NAME} >> ${WORKSPACE}/${env.DEV_TEST_ARTIFACTS}/milvus_${BINARY_VERSION}_mysql_dev_test.log"
|
||||
// sh "pytest test_restart.py --alluredir=\"test_out/dev/single/mysql\" --level=3 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local --service ${env.HELM_RELEASE_NAME}"
|
||||
if (isTimeTriggeredBuild()) {
|
||||
sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=2 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local --service ${env.HELM_RELEASE_NAME} >> ${WORKSPACE}/${env.DEV_TEST_ARTIFACTS}/milvus_${BINARY_VERSION}_mysql_dev_test.log"
|
||||
} else {
|
||||
sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local --service ${env.HELM_RELEASE_NAME} >> ${WORKSPACE}/${env.DEV_TEST_ARTIFACTS}/milvus_${BINARY_VERSION}_mysql_dev_test.log"
|
||||
}
|
||||
}
|
||||
|
||||
if (isTimeTriggeredBuild()) {
|
||||
// sqlite database backend test
|
||||
load "ci/jenkins/step/cleanupSingleDev.groovy"
|
||||
|
||||
retry(3) {
|
||||
try {
|
||||
dir ("milvus-helm/charts/milvus") {
|
||||
sh "helm install --wait --timeout 300s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP --set image.resources.requests.memory=8Gi --set image.resources.requests.cpu=2.0 --set image.resources.limits.memory=12Gi --set image.resources.limits.cpu=4.0 -f ci/db_backend/sqlite_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ."
|
||||
}
|
||||
} catch (exc) {
|
||||
def helmStatusCMD = "helm get manifest --namespace milvus ${env.HELM_RELEASE_NAME} | kubectl describe -n milvus -f - && \
|
||||
kubectl logs --namespace milvus -l \"app=milvus,release=${env.HELM_RELEASE_NAME}\" -c milvus && \
|
||||
helm status -n milvus ${env.HELM_RELEASE_NAME}"
|
||||
def helmResult = sh script: helmStatusCMD, returnStatus: true
|
||||
if (!helmResult) {
|
||||
sh "helm uninstall -n milvus ${env.HELM_RELEASE_NAME} && sleep 1m"
|
||||
}
|
||||
throw exc
|
||||
}
|
||||
}
|
||||
dir ("tests/milvus_python_test") {
|
||||
sh "pytest . --level=2 --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local >> ${WORKSPACE}/${env.DEV_TEST_ARTIFACTS}/milvus_${BINARY_VERSION}_sqlite_dev_test.log"
|
||||
sh "pytest . --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local --port=19121 --handler=HTTP >> ${WORKSPACE}/${env.DEV_TEST_ARTIFACTS}/milvus_${BINARY_VERSION}_sqlite_http_dev_test.log"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
boolean isTimeTriggeredBuild() {
|
||||
return (currentBuild.getBuildCauses('hudson.triggers.TimerTrigger$TimerTriggerCause').size() != 0) ? true : false;
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user