mirror of
https://gitee.com/milvus-io/milvus.git
synced 2025-12-29 06:55:27 +08:00
Refactor CI workflow (#1782)
* Set helm deploy timeout to 600s * Refactor CI workflow Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Refactor CI workflow Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Refactor CI workflow Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com> * Refactor CI workflow Signed-off-by: quicksilver <zhifeng.zhang@zilliz.com>
This commit is contained in:
quicksilver
GitHub
parent
a459a1efdd
commit
b995ccfb56
215
ci/jenkins/Jenkinsfile
vendored
215
ci/jenkins/Jenkinsfile
vendored
@ -34,7 +34,7 @@ pipeline {
|
||||
}
|
||||
|
||||
stages {
|
||||
stage ('Milvus Build and Test') {
|
||||
stage ('Milvus CI') {
|
||||
matrix {
|
||||
agent none
|
||||
axes {
|
||||
@ -53,148 +53,133 @@ pipeline {
|
||||
values 'gpu', 'cpu'
|
||||
}
|
||||
}
|
||||
environment {
|
||||
PACKAGE_VERSION = VersionNumber([
|
||||
versionNumberString : '${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}'
|
||||
]);
|
||||
DOCKER_VERSION = "${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}"
|
||||
}
|
||||
|
||||
stages {
|
||||
stage("Milvus CI") {
|
||||
environment {
|
||||
PACKAGE_VERSION = VersionNumber([
|
||||
versionNumberString : '${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}'
|
||||
]);
|
||||
DOCKER_VERSION = "${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}"
|
||||
stage("Milvus Build and Unittest") {
|
||||
agent {
|
||||
kubernetes {
|
||||
label "${OS_NAME}-${BINARY_VERSION}-build-${SEMVER}-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"
|
||||
defaultContainer 'jnlp'
|
||||
yamlFile "ci/jenkins/pod/milvus-${BINARY_VERSION}-version-${OS_NAME}-build-env-pod.yaml"
|
||||
}
|
||||
}
|
||||
|
||||
stages {
|
||||
stage('Run Build') {
|
||||
agent {
|
||||
kubernetes {
|
||||
label "${OS_NAME}-${BINARY_VERSION}-build-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"
|
||||
defaultContainer 'jnlp'
|
||||
yamlFile "ci/jenkins/pod/milvus-${BINARY_VERSION}-version-${OS_NAME}-build-env-pod.yaml"
|
||||
stage('Build') {
|
||||
steps {
|
||||
container("milvus-${BINARY_VERSION}-build-env") {
|
||||
script {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/build.groovy"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stages {
|
||||
stage('Build') {
|
||||
steps {
|
||||
container("milvus-${BINARY_VERSION}-build-env") {
|
||||
script {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/build.groovy"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Unittest') {
|
||||
steps {
|
||||
container("milvus-${BINARY_VERSION}-build-env") {
|
||||
script {
|
||||
if ("${BINARY_VERSION}" == "gpu") {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/unittest.groovy"
|
||||
} else {
|
||||
echo "Skip Unittest"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Code Coverage') {
|
||||
steps {
|
||||
container("milvus-${BINARY_VERSION}-build-env") {
|
||||
script {
|
||||
if ("${BINARY_VERSION}" == "gpu") {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy"
|
||||
} else {
|
||||
echo "Skip Code Coverage"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Upload Package') {
|
||||
steps {
|
||||
container("milvus-${BINARY_VERSION}-build-env") {
|
||||
script {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/package.groovy"
|
||||
}
|
||||
stage('Unittest') {
|
||||
steps {
|
||||
container("milvus-${BINARY_VERSION}-build-env") {
|
||||
script {
|
||||
if ("${BINARY_VERSION}" == "gpu") {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/unittest.groovy"
|
||||
} else {
|
||||
echo "Skip Unittest"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Publish docker images') {
|
||||
agent {
|
||||
kubernetes {
|
||||
label "${OS_NAME}-${BINARY_VERSION}-publish-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"
|
||||
defaultContainer 'jnlp'
|
||||
yamlFile 'ci/jenkins/pod/docker-pod.yaml'
|
||||
}
|
||||
}
|
||||
|
||||
stages {
|
||||
stage('Publish') {
|
||||
steps {
|
||||
container('publish-images') {
|
||||
script {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy"
|
||||
}
|
||||
stage('Code Coverage') {
|
||||
steps {
|
||||
container("milvus-${BINARY_VERSION}-build-env") {
|
||||
script {
|
||||
if ("${BINARY_VERSION}" == "gpu") {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy"
|
||||
} else {
|
||||
echo "Skip Code Coverage"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Deploy to Development') {
|
||||
environment {
|
||||
FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-").replaceAll("_", "-")
|
||||
FORMAT_OS_NAME = "${OS_NAME}".replaceAll("\\.", "-").replaceAll("_", "-")
|
||||
HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${FORMAT_OS_NAME}-${BINARY_VERSION}".toLowerCase()
|
||||
}
|
||||
|
||||
agent {
|
||||
kubernetes {
|
||||
label "${OS_NAME}-${BINARY_VERSION}-dev-test-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"
|
||||
defaultContainer 'jnlp'
|
||||
yamlFile 'ci/jenkins/pod/testEnvironment.yaml'
|
||||
}
|
||||
}
|
||||
|
||||
stages {
|
||||
stage('Deploy to Dev') {
|
||||
steps {
|
||||
container('milvus-test-env') {
|
||||
script {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/deploySingle2Dev.groovy"
|
||||
}
|
||||
}
|
||||
stage('Upload Package') {
|
||||
steps {
|
||||
container("milvus-${BINARY_VERSION}-build-env") {
|
||||
script {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/package.groovy"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Dev Test') {
|
||||
steps {
|
||||
container('milvus-test-env') {
|
||||
script {
|
||||
boolean isNightlyTest = isTimeTriggeredBuild()
|
||||
if (isNightlyTest) {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy"
|
||||
} else {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy"
|
||||
}
|
||||
}
|
||||
stage('Publish Docker images') {
|
||||
agent {
|
||||
kubernetes {
|
||||
label "${OS_NAME}-${BINARY_VERSION}-publish-${SEMVER}-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"
|
||||
defaultContainer 'jnlp'
|
||||
yamlFile 'ci/jenkins/pod/docker-pod.yaml'
|
||||
}
|
||||
}
|
||||
|
||||
stages {
|
||||
stage('Publish') {
|
||||
steps {
|
||||
container('publish-images') {
|
||||
script {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
stage('Deploy to Development') {
|
||||
environment {
|
||||
FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-").replaceAll("_", "-")
|
||||
FORMAT_OS_NAME = "${OS_NAME}".replaceAll("\\.", "-").replaceAll("_", "-")
|
||||
HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${FORMAT_OS_NAME}-${BINARY_VERSION}".toLowerCase()
|
||||
}
|
||||
|
||||
agent {
|
||||
kubernetes {
|
||||
label "${OS_NAME}-${BINARY_VERSION}-dev-test-${SEMVER}-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}"
|
||||
defaultContainer 'jnlp'
|
||||
yamlFile 'ci/jenkins/pod/testEnvironment.yaml'
|
||||
}
|
||||
}
|
||||
|
||||
stages {
|
||||
stage('Dev Test') {
|
||||
steps {
|
||||
container('milvus-test-env') {
|
||||
script {
|
||||
boolean isNightlyTest = isTimeTriggeredBuild()
|
||||
if (isNightlyTest) {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy"
|
||||
} else {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
post {
|
||||
cleanup {
|
||||
container('milvus-test-env') {
|
||||
script {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy"
|
||||
}
|
||||
}
|
||||
post {
|
||||
cleanup {
|
||||
container('milvus-test-env') {
|
||||
script {
|
||||
load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -36,6 +36,13 @@ spec:
|
||||
ports:
|
||||
- containerPort: 3306
|
||||
name: mysql
|
||||
resources:
|
||||
limits:
|
||||
memory: "500Mi"
|
||||
cpu: "0.5"
|
||||
requests:
|
||||
memory: "200Mi"
|
||||
cpu: "0.2"
|
||||
tolerations:
|
||||
- key: dedicated
|
||||
operator: Equal
|
||||
|
||||
@ -36,6 +36,13 @@ spec:
|
||||
ports:
|
||||
- containerPort: 3306
|
||||
name: mysql
|
||||
resources:
|
||||
limits:
|
||||
memory: "500Mi"
|
||||
cpu: "0.5"
|
||||
requests:
|
||||
memory: "200Mi"
|
||||
cpu: "0.2"
|
||||
tolerations:
|
||||
- key: dedicated
|
||||
operator: Equal
|
||||
|
||||
@ -38,6 +38,13 @@ spec:
|
||||
ports:
|
||||
- containerPort: 3306
|
||||
name: mysql
|
||||
resources:
|
||||
limits:
|
||||
memory: "500Mi"
|
||||
cpu: "0.5"
|
||||
requests:
|
||||
memory: "200Mi"
|
||||
cpu: "0.2"
|
||||
tolerations:
|
||||
- key: dedicated
|
||||
operator: Equal
|
||||
|
||||
@ -38,6 +38,13 @@ spec:
|
||||
ports:
|
||||
- containerPort: 3306
|
||||
name: mysql
|
||||
resources:
|
||||
limits:
|
||||
memory: "500Mi"
|
||||
cpu: "0.5"
|
||||
requests:
|
||||
memory: "200Mi"
|
||||
cpu: "0.2"
|
||||
tolerations:
|
||||
- key: dedicated
|
||||
operator: Equal
|
||||
|
||||
@ -1,7 +0,0 @@
|
||||
sh 'helm version'
|
||||
sh 'helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
|
||||
sh 'helm repo update'
|
||||
dir ('milvus-helm') {
|
||||
checkout([$class: 'GitSCM', branches: [[name: "master"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/master:refs/remotes/origin/master"]]])
|
||||
sh "helm install --wait --timeout 600s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ."
|
||||
}
|
||||
@ -1,9 +1,17 @@
|
||||
dir ('milvus-helm') {
|
||||
sh 'helm version'
|
||||
sh 'helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
|
||||
sh 'helm repo update'
|
||||
checkout([$class: 'GitSCM', branches: [[name: "master"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/master:refs/remotes/origin/master"]]])
|
||||
sh "helm install --wait --timeout 600s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ."
|
||||
}
|
||||
|
||||
dir ("tests/milvus_python_test") {
|
||||
// sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
|
||||
sh 'python3 -m pip install -r requirements.txt'
|
||||
sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local"
|
||||
sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local"
|
||||
}
|
||||
// mysql database backend test
|
||||
// sqlite database backend test
|
||||
load "ci/jenkins/step/cleanupSingleDev.groovy"
|
||||
|
||||
if (!fileExists('milvus-helm')) {
|
||||
@ -12,8 +20,8 @@ if (!fileExists('milvus-helm')) {
|
||||
}
|
||||
}
|
||||
dir ("milvus-helm") {
|
||||
sh "helm install --wait --timeout 300s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ."
|
||||
sh "helm install --wait --timeout 300s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/sqlite_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ."
|
||||
}
|
||||
dir ("tests/milvus_python_test") {
|
||||
sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local"
|
||||
sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local"
|
||||
}
|
||||
|
||||
@ -1,5 +1,13 @@
|
||||
dir ('milvus-helm') {
|
||||
sh 'helm version'
|
||||
sh 'helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts'
|
||||
sh 'helm repo update'
|
||||
checkout([$class: 'GitSCM', branches: [[name: "master"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/master:refs/remotes/origin/master"]]])
|
||||
sh "helm install --wait --timeout 600s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ."
|
||||
}
|
||||
|
||||
dir ("tests/milvus_python_test") {
|
||||
// sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com'
|
||||
sh 'python3 -m pip install -r requirements.txt'
|
||||
sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local"
|
||||
sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local"
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user