diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 98af7ee4d9..a1f23fbba9 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -34,7 +34,7 @@ pipeline { } stages { - stage ('Milvus Build and Test') { + stage ('Milvus CI') { matrix { agent none axes { @@ -53,148 +53,133 @@ pipeline { values 'gpu', 'cpu' } } + environment { + PACKAGE_VERSION = VersionNumber([ + versionNumberString : '${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' + ]); + DOCKER_VERSION = "${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}" + } stages { - stage("Milvus CI") { - environment { - PACKAGE_VERSION = VersionNumber([ - versionNumberString : '${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' - ]); - DOCKER_VERSION = "${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}" + stage("Milvus Build and Unittest") { + agent { + kubernetes { + label "${OS_NAME}-${BINARY_VERSION}-build-${SEMVER}-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}" + defaultContainer 'jnlp' + yamlFile "ci/jenkins/pod/milvus-${BINARY_VERSION}-version-${OS_NAME}-build-env-pod.yaml" + } } stages { - stage('Run Build') { - agent { - kubernetes { - label "${OS_NAME}-${BINARY_VERSION}-build-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}" - defaultContainer 'jnlp' - yamlFile "ci/jenkins/pod/milvus-${BINARY_VERSION}-version-${OS_NAME}-build-env-pod.yaml" + stage('Build') { + steps { + container("milvus-${BINARY_VERSION}-build-env") { + script { + load "${env.WORKSPACE}/ci/jenkins/step/build.groovy" + } } } + } - stages { - stage('Build') { - steps { - container("milvus-${BINARY_VERSION}-build-env") { - script { - load "${env.WORKSPACE}/ci/jenkins/step/build.groovy" - } - } - } - } - - stage('Unittest') { - steps { - container("milvus-${BINARY_VERSION}-build-env") { - script { - if ("${BINARY_VERSION}" == "gpu") { - load "${env.WORKSPACE}/ci/jenkins/step/unittest.groovy" - } else { - echo "Skip Unittest" - } - } - } - } - } - - stage('Code Coverage') { - steps { - container("milvus-${BINARY_VERSION}-build-env") { - script { - if ("${BINARY_VERSION}" == "gpu") { - load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" - } else { - echo "Skip Code Coverage" - } - } - } - } - } - - stage('Upload Package') { - steps { - container("milvus-${BINARY_VERSION}-build-env") { - script { - load "${env.WORKSPACE}/ci/jenkins/step/package.groovy" - } + stage('Unittest') { + steps { + container("milvus-${BINARY_VERSION}-build-env") { + script { + if ("${BINARY_VERSION}" == "gpu") { + load "${env.WORKSPACE}/ci/jenkins/step/unittest.groovy" + } else { + echo "Skip Unittest" } } } } } - stage('Publish docker images') { - agent { - kubernetes { - label "${OS_NAME}-${BINARY_VERSION}-publish-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}" - defaultContainer 'jnlp' - yamlFile 'ci/jenkins/pod/docker-pod.yaml' - } - } - - stages { - stage('Publish') { - steps { - container('publish-images') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy" - } + stage('Code Coverage') { + steps { + container("milvus-${BINARY_VERSION}-build-env") { + script { + if ("${BINARY_VERSION}" == "gpu") { + load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" + } else { + echo "Skip Code Coverage" } } } } } - stage('Deploy to Development') { - environment { - FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-").replaceAll("_", "-") - FORMAT_OS_NAME = "${OS_NAME}".replaceAll("\\.", "-").replaceAll("_", "-") - HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${FORMAT_OS_NAME}-${BINARY_VERSION}".toLowerCase() - } - - agent { - kubernetes { - label "${OS_NAME}-${BINARY_VERSION}-dev-test-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}" - defaultContainer 'jnlp' - yamlFile 'ci/jenkins/pod/testEnvironment.yaml' - } - } - - stages { - stage('Deploy to Dev') { - steps { - container('milvus-test-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/deploySingle2Dev.groovy" - } - } + stage('Upload Package') { + steps { + container("milvus-${BINARY_VERSION}-build-env") { + script { + load "${env.WORKSPACE}/ci/jenkins/step/package.groovy" } } + } + } + } + } - stage('Dev Test') { - steps { - container('milvus-test-env') { - script { - boolean isNightlyTest = isTimeTriggeredBuild() - if (isNightlyTest) { - load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy" - } else { - load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy" - } - } + stage('Publish Docker images') { + agent { + kubernetes { + label "${OS_NAME}-${BINARY_VERSION}-publish-${SEMVER}-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}" + defaultContainer 'jnlp' + yamlFile 'ci/jenkins/pod/docker-pod.yaml' + } + } + + stages { + stage('Publish') { + steps { + container('publish-images') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy" + } + } + } + } + } + } + + stage('Deploy to Development') { + environment { + FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-").replaceAll("_", "-") + FORMAT_OS_NAME = "${OS_NAME}".replaceAll("\\.", "-").replaceAll("_", "-") + HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${FORMAT_OS_NAME}-${BINARY_VERSION}".toLowerCase() + } + + agent { + kubernetes { + label "${OS_NAME}-${BINARY_VERSION}-dev-test-${SEMVER}-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}" + defaultContainer 'jnlp' + yamlFile 'ci/jenkins/pod/testEnvironment.yaml' + } + } + + stages { + stage('Dev Test') { + steps { + container('milvus-test-env') { + script { + boolean isNightlyTest = isTimeTriggeredBuild() + if (isNightlyTest) { + load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy" + } else { + load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy" } } } } + } + } - post { - cleanup { - container('milvus-test-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" - } - } + post { + cleanup { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" } } } diff --git a/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml b/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml index 6a400883db..3a765bb583 100644 --- a/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml @@ -36,6 +36,13 @@ spec: ports: - containerPort: 3306 name: mysql + resources: + limits: + memory: "500Mi" + cpu: "0.5" + requests: + memory: "200Mi" + cpu: "0.2" tolerations: - key: dedicated operator: Equal diff --git a/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml b/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml index 34474b7246..6758c682d1 100644 --- a/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml @@ -36,6 +36,13 @@ spec: ports: - containerPort: 3306 name: mysql + resources: + limits: + memory: "500Mi" + cpu: "0.5" + requests: + memory: "200Mi" + cpu: "0.2" tolerations: - key: dedicated operator: Equal diff --git a/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml b/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml index 608e9b32de..57fd514fd2 100644 --- a/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml @@ -38,6 +38,13 @@ spec: ports: - containerPort: 3306 name: mysql + resources: + limits: + memory: "500Mi" + cpu: "0.5" + requests: + memory: "200Mi" + cpu: "0.2" tolerations: - key: dedicated operator: Equal diff --git a/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml b/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml index b1d3a74aa0..cc9812fb67 100644 --- a/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml @@ -38,6 +38,13 @@ spec: ports: - containerPort: 3306 name: mysql + resources: + limits: + memory: "500Mi" + cpu: "0.5" + requests: + memory: "200Mi" + cpu: "0.2" tolerations: - key: dedicated operator: Equal diff --git a/ci/jenkins/step/deploySingle2Dev.groovy b/ci/jenkins/step/deploySingle2Dev.groovy deleted file mode 100644 index 8cefbf226a..0000000000 --- a/ci/jenkins/step/deploySingle2Dev.groovy +++ /dev/null @@ -1,7 +0,0 @@ -sh 'helm version' -sh 'helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' -sh 'helm repo update' -dir ('milvus-helm') { - checkout([$class: 'GitSCM', branches: [[name: "master"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/master:refs/remotes/origin/master"]]]) - sh "helm install --wait --timeout 600s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." -} diff --git a/ci/jenkins/step/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy index 38c762c8df..75eb89be63 100644 --- a/ci/jenkins/step/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -1,9 +1,17 @@ +dir ('milvus-helm') { + sh 'helm version' + sh 'helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' + sh 'helm repo update' + checkout([$class: 'GitSCM', branches: [[name: "master"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/master:refs/remotes/origin/master"]]]) + sh "helm install --wait --timeout 600s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." +} + dir ("tests/milvus_python_test") { // sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' sh 'python3 -m pip install -r requirements.txt' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local" } -// mysql database backend test +// sqlite database backend test load "ci/jenkins/step/cleanupSingleDev.groovy" if (!fileExists('milvus-helm')) { @@ -12,8 +20,8 @@ if (!fileExists('milvus-helm')) { } } dir ("milvus-helm") { - sh "helm install --wait --timeout 300s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." + sh "helm install --wait --timeout 300s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/sqlite_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." } dir ("tests/milvus_python_test") { - sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local" } diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index b5e33a4260..f1f07bef2a 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -1,5 +1,13 @@ +dir ('milvus-helm') { + sh 'helm version' + sh 'helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' + sh 'helm repo update' + checkout([$class: 'GitSCM', branches: [[name: "master"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/master:refs/remotes/origin/master"]]]) + sh "helm install --wait --timeout 600s --set image.repository=registry.zilliz.com/milvus/engine --set image.tag=${DOCKER_VERSION} --set image.pullPolicy=Always --set service.type=ClusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." +} + dir ("tests/milvus_python_test") { // sh 'python3 -m pip install -r requirements.txt -i http://pypi.douban.com/simple --trusted-host pypi.douban.com' sh 'python3 -m pip install -r requirements.txt' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.HELM_RELEASE_NAME}.milvus.svc.cluster.local" }