From 7e2f364596e1aa07152010452b5640674fd6480f Mon Sep 17 00:00:00 2001 From: test Date: Wed, 16 Oct 2019 21:20:21 +0800 Subject: [PATCH 01/13] add ci Former-commit-id: 6ba5e9408d396d55b15d2ff1d82407e4b7169af9 --- ci/jenkins/Jenkinsfile | 142 ++++++++++++++++++ ci/jenkins/jenkinsfile/build.groovy | 11 ++ .../jenkinsfile/cleanupSingleDev.groovy | 10 ++ .../jenkinsfile/deploySingle2Dev.groovy | 14 ++ ci/jenkins/jenkinsfile/package.groovy | 15 ++ ci/jenkins/jenkinsfile/publishImages.groovy | 27 ++++ ci/jenkins/jenkinsfile/singleDevTest.groovy | 22 +++ ci/jenkins/pod/docker-pod.yaml | 23 +++ ci/jenkins/pod/milvus-build-env-pod.yaml | 22 +++ ci/jenkins/pod/testEnvironment.yaml | 22 +++ docker/deploy/ubuntu16.04/Dockerfile | 23 +++ .../deploy/ubuntu16.04/docker-entrypoint.sh | 10 ++ 12 files changed, 341 insertions(+) create mode 100644 ci/jenkins/Jenkinsfile create mode 100644 ci/jenkins/jenkinsfile/build.groovy create mode 100644 ci/jenkins/jenkinsfile/cleanupSingleDev.groovy create mode 100644 ci/jenkins/jenkinsfile/deploySingle2Dev.groovy create mode 100644 ci/jenkins/jenkinsfile/package.groovy create mode 100644 ci/jenkins/jenkinsfile/publishImages.groovy create mode 100644 ci/jenkins/jenkinsfile/singleDevTest.groovy create mode 100644 ci/jenkins/pod/docker-pod.yaml create mode 100644 ci/jenkins/pod/milvus-build-env-pod.yaml create mode 100644 ci/jenkins/pod/testEnvironment.yaml create mode 100644 docker/deploy/ubuntu16.04/Dockerfile create mode 100755 docker/deploy/ubuntu16.04/docker-entrypoint.sh diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile new file mode 100644 index 0000000000..c035627bc9 --- /dev/null +++ b/ci/jenkins/Jenkinsfile @@ -0,0 +1,142 @@ +pipeline { + agent none + + options { + timestamps() + } + + parameters{ + choice choices: ['Release', 'Debug'], description: '', name: 'BUILD_TYPE' + string defaultValue: 'cf1434e7-5a4b-4d25-82e8-88d667aef9e5', description: 'GIT CREDENTIALS ID', name: 'GIT_CREDENTIALS_ID', trim: true + string defaultValue: '1a527823-d2b7-44fd-834b-9844350baf14', description: 'JFROG CREDENTIALS ID', name: 'JFROG_CREDENTIALS_ID', trim: true + string defaultValue: 'ba070c98-c8cc-4f7c-b657-897715f359fc', description: 'DOCKER CREDENTIALS ID', name: 'DOCKER_CREDENTIALS_ID', trim: true + string defaultValue: 'http://192.168.1.202/artifactory/milvus', description: 'JFROG ARTFACTORY URL', name: 'JFROG_ARTFACTORY_URL', trim: true + } + + environment { + PROJECT_NAME = "milvus" + LOWER_BUILD_TYPE = params.BUILD_TYPE.toLowerCase() + SEMVER = "${BRANCH_NAME}" + JOBNAMES = env.JOB_NAME.split('/') + PIPELIEN_NAME = "${JOBNAMES[0]}" + } + + stages { + stage("Ubuntu 16.04") { + environment { + OS_NAME = "ubuntu16.04" + PACKAGE_VERSION = VersionNumber([ + versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-${OS_NAME}-x86_64-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' + ]); + DOCKER_VERSION = "${SEMVER}-${OS_NAME}-${LOWER_BUILD_TYPE}" + } + + stages { + stage("Run Build") { + agent { + kubernetes { + label 'build' + defaultContainer 'jnlp' + yamlFile 'ci/pod/milvus-build-env-pod.yaml' + } + } + + stages { + stage('Build') { + steps { + container('milvus-build-env') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/build.groovy" + } + } + } + } + stage('Upload Package') { + steps { + container('milvus-build-env') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/package.groovy" + } + } + } + } + } + } + + stage("Publish docker images") { + agent { + kubernetes { + label 'publish' + defaultContainer 'jnlp' + yamlFile 'ci/pod/docker-pod.yaml' + } + } + + stages { + stage('Publish') { + steps { + container('publish-images'){ + script { + load "${env.WORKSPACE}/ci/jenkinsfile/publishImages.groovy" + } + } + } + } + } + } + + stage("Deploy to Development") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yamlFile 'ci/pod/testEnvironment.yaml' + } + } + + stages { + stage("Deploy to Dev") { + steps { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/deploySingle2Dev.groovy" + } + } + } + } + + stage("Dev Test") { + steps { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/singleDevTest.groovy" + } + } + } + } + + stage ("Cleanup Dev") { + steps { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanupSingleDev.groovy" + } + } + } + } + } + post { + unsuccessful { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkinsfile/cleanupSingleDev.groovy" + } + } + } + } + } + } + } + } +} + diff --git a/ci/jenkins/jenkinsfile/build.groovy b/ci/jenkins/jenkinsfile/build.groovy new file mode 100644 index 0000000000..f47397038d --- /dev/null +++ b/ci/jenkins/jenkinsfile/build.groovy @@ -0,0 +1,11 @@ +timeout(time: 60, unit: 'MINUTES') { + dir ("core") { + sh "git config --global user.email \"test@zilliz.com\"" + sh "git config --global user.name \"test\"" + // sh "./build.sh -l" + withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { + sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -j -u" + } + } +} + diff --git a/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy b/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy new file mode 100644 index 0000000000..b899e9742d --- /dev/null +++ b/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy @@ -0,0 +1,10 @@ +try { + def jobNames = env.JOB_NAME.split('/') + sh "helm del --purge ${env.PIPELIEN_NAME}-${env.BUILD_NUMBER}-single-gpu" +} catch (exc) { + def helmResult = sh script: "helm status ${env.PIPELIEN_NAME}-${env.BUILD_NUMBER}-single-gpu", returnStatus: true + if (!helmResult) { + sh "helm del --purge ${env.PIPELIEN_NAME}-${env.BUILD_NUMBER}-single-gpu" + } + throw exc +} diff --git a/ci/jenkins/jenkinsfile/deploySingle2Dev.groovy b/ci/jenkins/jenkinsfile/deploySingle2Dev.groovy new file mode 100644 index 0000000000..87ded45255 --- /dev/null +++ b/ci/jenkins/jenkinsfile/deploySingle2Dev.groovy @@ -0,0 +1,14 @@ +try { + sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' + sh 'helm repo update' + dir ('milvus-helm') { + checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}", url: "https://github.com/zilliz-ci-robot/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]]) + dir ("milvus-gpu") { + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELIEN_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/values.yaml --namespace milvus ." + } + } +} catch (exc) { + echo 'Helm running failed!' + sh "helm del --purge ${env.PIPELIEN_NAME}-${env.BUILD_NUMBER}-single-gpu" + throw exc +} diff --git a/ci/jenkins/jenkinsfile/package.groovy b/ci/jenkins/jenkinsfile/package.groovy new file mode 100644 index 0000000000..aa969e9ad5 --- /dev/null +++ b/ci/jenkins/jenkinsfile/package.groovy @@ -0,0 +1,15 @@ +timeout(time: 5, unit: 'MINUTES') { + dir ("core") { + if (fileExists('milvus')) { + sh "tar -zcvf ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz ./milvus" + withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { + def uploadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -T ./${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz ${params.JFROG_ARTFACTORY_URL}/milvus/package/${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz") + if (uploadStatus != 0) { + error("\" ${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz \" upload to \" ${params.JFROG_ARTFACTORY_URL}/milvus/package/${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz \" failed!") + } + } + } else { + error("Milvus binary directory don't exists!") + } + } +} diff --git a/ci/jenkins/jenkinsfile/publishImages.groovy b/ci/jenkins/jenkinsfile/publishImages.groovy new file mode 100644 index 0000000000..5d19ab5a99 --- /dev/null +++ b/ci/jenkins/jenkinsfile/publishImages.groovy @@ -0,0 +1,27 @@ +container('publish-images') { + timeout(time: 15, unit: 'MINUTES') { + dir ("docker/deploy/${OS_NAME}") { + def binaryPackage = "${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz" + withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { + def downloadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -O ${params.JFROG_ARTFACTORY_URL}/milvus/package/${binaryPackage}") + + if (downloadStatus != 0) { + error("\" Download \" ${params.JFROG_ARTFACTORY_URL}/milvus/package/${binaryPackage} \" failed!") + } + } + sh "tar zxvf ${binaryPackage}" + def imageName = "${PROJECT_NAME}/engine:${DOCKER_VERSION}" + + try { + def customImage = docker.build("${imageName}") + docker.withRegistry('https://registry.zilliz.com', "${params.DOCKER_CREDENTIALS_ID}") { + customImage.push() + } + } catch (exc) { + throw exc + } finally { + sh "docker rmi ${imageName}" + } + } + } +} diff --git a/ci/jenkins/jenkinsfile/singleDevTest.groovy b/ci/jenkins/jenkinsfile/singleDevTest.groovy new file mode 100644 index 0000000000..6871c38ecc --- /dev/null +++ b/ci/jenkins/jenkinsfile/singleDevTest.groovy @@ -0,0 +1,22 @@ +timeout(time: 30, unit: 'MINUTES') { + dir ("tests/milvus_python_test") { + sh 'python3 -m pip install -r requirements.txt' + sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.PIPELIEN_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local" + } + // mysql database backend test + load "${env.WORKSPACE}/ci/jenkinsfile/cleanupSingleDev.groovy" + + if (!fileExists('milvus-helm')) { + dir ("milvus-helm") { + checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}", url: "https://github.com/zilliz-ci-robot/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]]) + } + } + dir ("milvus-helm") { + dir ("milvus-gpu") { + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELIEN_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/db_backend/mysql_values.yaml --namespace milvus ." + } + } + dir ("tests/milvus_python_test") { + sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.PIPELIEN_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local" + } +} diff --git a/ci/jenkins/pod/docker-pod.yaml b/ci/jenkins/pod/docker-pod.yaml new file mode 100644 index 0000000000..a4fed0bcad --- /dev/null +++ b/ci/jenkins/pod/docker-pod.yaml @@ -0,0 +1,23 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: publish + componet: docker +spec: + containers: + - name: publish-images + image: registry.zilliz.com/library/docker:v1.0.0 + securityContext: + privileged: true + command: + - cat + tty: true + volumeMounts: + - name: docker-sock + mountPath: /var/run/docker.sock + volumes: + - name: docker-sock + hostPath: + path: /var/run/docker.sock + diff --git a/ci/jenkins/pod/milvus-build-env-pod.yaml b/ci/jenkins/pod/milvus-build-env-pod.yaml new file mode 100644 index 0000000000..358cdee19c --- /dev/null +++ b/ci/jenkins/pod/milvus-build-env-pod.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Pod +metadata: + name: milvus-build-env + labels: + app: milvus + componet: build-env +spec: + containers: + - name: milvus-build-env + image: registry.zilliz.com/milvus/milvus-build-env:v0.5.0 + command: + - cat + tty: true + resources: + limits: + memory: "32Gi" + cpu: "8.0" + nvidia.com/gpu: 1 + requests: + memory: "16Gi" + cpu: "4.0" diff --git a/ci/jenkins/pod/testEnvironment.yaml b/ci/jenkins/pod/testEnvironment.yaml new file mode 100644 index 0000000000..174277dfcc --- /dev/null +++ b/ci/jenkins/pod/testEnvironment.yaml @@ -0,0 +1,22 @@ +apiVersion: v1 +kind: Pod +metadata: + labels: + app: milvus + componet: test-env +spec: + containers: + - name: milvus-test-env + image: registry.zilliz.com/milvus/milvus-test-env:v0.1 + command: + - cat + tty: true + volumeMounts: + - name: kubeconf + mountPath: /root/.kube/ + readOnly: true + volumes: + - name: kubeconf + secret: + secretName: test-cluster-config + diff --git a/docker/deploy/ubuntu16.04/Dockerfile b/docker/deploy/ubuntu16.04/Dockerfile new file mode 100644 index 0000000000..c5ca0ab03e --- /dev/null +++ b/docker/deploy/ubuntu16.04/Dockerfile @@ -0,0 +1,23 @@ +FROM nvidia/cuda:10.1-devel-ubuntu16.04 + +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility + +RUN rm -rf /etc/apt/sources.list.d/nvidia-ml.list && rm -rf /etc/apt/sources.list.d/cuda.list + +RUN apt-get update && apt-get install -y --no-install-recommends \ + gfortran libsqlite3-dev libmysqlclient-dev libcurl4-openssl-dev python3 && \ + apt-get remove --purge -y && \ + rm -rf /var/lib/apt/lists/* + +RUN ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so + +COPY ./docker-entrypoint.sh /opt +COPY ./milvus /opt/milvus +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/milvus/lib" + +ENTRYPOINT [ "/opt/docker-entrypoint.sh" ] + +CMD [ "start" ] + +EXPOSE 19530 + diff --git a/docker/deploy/ubuntu16.04/docker-entrypoint.sh b/docker/deploy/ubuntu16.04/docker-entrypoint.sh new file mode 100755 index 0000000000..446c174d74 --- /dev/null +++ b/docker/deploy/ubuntu16.04/docker-entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +if [ "$1" == 'start' ]; then + cd /opt/milvus/scripts && ./start_server.sh +fi + +exec "$@" + From c6970dac2a9a0cbe78b63f7b77e2f011e0cadf88 Mon Sep 17 00:00:00 2001 From: test Date: Wed, 16 Oct 2019 21:25:02 +0800 Subject: [PATCH 02/13] update ci/jenkins/Jenkinsfile Former-commit-id: 93f8729ec00b1e5c98b11c730aea684e1379af7a --- ci/jenkins/Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index c035627bc9..dd94ec6dde 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -37,7 +37,7 @@ pipeline { kubernetes { label 'build' defaultContainer 'jnlp' - yamlFile 'ci/pod/milvus-build-env-pod.yaml' + yamlFile 'ci/jenkins/pod/milvus-build-env-pod.yaml' } } @@ -68,7 +68,7 @@ pipeline { kubernetes { label 'publish' defaultContainer 'jnlp' - yamlFile 'ci/pod/docker-pod.yaml' + yamlFile 'ci/jenkins/pod/docker-pod.yaml' } } @@ -90,7 +90,7 @@ pipeline { kubernetes { label 'dev-test' defaultContainer 'jnlp' - yamlFile 'ci/pod/testEnvironment.yaml' + yamlFile 'ci/jenkins/pod/testEnvironment.yaml' } } From 7bc08900cf1902600aa8fddce1cf4c3037f8c4ed Mon Sep 17 00:00:00 2001 From: test Date: Wed, 16 Oct 2019 21:31:40 +0800 Subject: [PATCH 03/13] update ci/jenkins/jenkinsfile/build.groovy Former-commit-id: f8ef19525a1596005552984c39aaa4c58ee71d9e --- ci/jenkins/jenkinsfile/build.groovy | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/jenkins/jenkinsfile/build.groovy b/ci/jenkins/jenkinsfile/build.groovy index f47397038d..9dda1feda4 100644 --- a/ci/jenkins/jenkinsfile/build.groovy +++ b/ci/jenkins/jenkinsfile/build.groovy @@ -2,9 +2,9 @@ timeout(time: 60, unit: 'MINUTES') { dir ("core") { sh "git config --global user.email \"test@zilliz.com\"" sh "git config --global user.name \"test\"" - // sh "./build.sh -l" + // sh "./build.sh -l" withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -j -u" + sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -d /opt/milvus -j -u" } } } From 5a7ed935dd5ad304ce920c48c763876666ed2ea6 Mon Sep 17 00:00:00 2001 From: test Date: Wed, 16 Oct 2019 21:37:51 +0800 Subject: [PATCH 04/13] update ci/jenkins/jenkinsfile/singleDevTest.groovy Former-commit-id: 792fe4cfeb149438c37b06f0705e7158777dafb0 --- ci/jenkins/Jenkinsfile | 14 +++++++------- ci/jenkins/jenkinsfile/singleDevTest.groovy | 2 +- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index dd94ec6dde..5f3227fce6 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -46,7 +46,7 @@ pipeline { steps { container('milvus-build-env') { script { - load "${env.WORKSPACE}/ci/jenkinsfile/build.groovy" + load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/build.groovy" } } } @@ -55,7 +55,7 @@ pipeline { steps { container('milvus-build-env') { script { - load "${env.WORKSPACE}/ci/jenkinsfile/package.groovy" + load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/package.groovy" } } } @@ -77,7 +77,7 @@ pipeline { steps { container('publish-images'){ script { - load "${env.WORKSPACE}/ci/jenkinsfile/publishImages.groovy" + load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/publishImages.groovy" } } } @@ -99,7 +99,7 @@ pipeline { steps { container('milvus-test-env') { script { - load "${env.WORKSPACE}/ci/jenkinsfile/deploySingle2Dev.groovy" + load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/deploySingle2Dev.groovy" } } } @@ -109,7 +109,7 @@ pipeline { steps { container('milvus-test-env') { script { - load "${env.WORKSPACE}/ci/jenkinsfile/singleDevTest.groovy" + load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/singleDevTest.groovy" } } } @@ -119,7 +119,7 @@ pipeline { steps { container('milvus-test-env') { script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanupSingleDev.groovy" + load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" } } } @@ -129,7 +129,7 @@ pipeline { unsuccessful { container('milvus-test-env') { script { - load "${env.WORKSPACE}/ci/jenkinsfile/cleanupSingleDev.groovy" + load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" } } } diff --git a/ci/jenkins/jenkinsfile/singleDevTest.groovy b/ci/jenkins/jenkinsfile/singleDevTest.groovy index 6871c38ecc..1022dafd67 100644 --- a/ci/jenkins/jenkinsfile/singleDevTest.groovy +++ b/ci/jenkins/jenkinsfile/singleDevTest.groovy @@ -4,7 +4,7 @@ timeout(time: 30, unit: 'MINUTES') { sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.PIPELIEN_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local" } // mysql database backend test - load "${env.WORKSPACE}/ci/jenkinsfile/cleanupSingleDev.groovy" + load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" if (!fileExists('milvus-helm')) { dir ("milvus-helm") { From 6408aed85b68764ed48c2eea1acc10015809e9fe Mon Sep 17 00:00:00 2001 From: test Date: Wed, 16 Oct 2019 21:43:41 +0800 Subject: [PATCH 05/13] open code style check Former-commit-id: ea434d51cb724a37b1a1bae80d92d1c846bf26e1 --- ci/jenkins/jenkinsfile/build.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/jenkins/jenkinsfile/build.groovy b/ci/jenkins/jenkinsfile/build.groovy index 9dda1feda4..c1420c32cb 100644 --- a/ci/jenkins/jenkinsfile/build.groovy +++ b/ci/jenkins/jenkinsfile/build.groovy @@ -2,7 +2,7 @@ timeout(time: 60, unit: 'MINUTES') { dir ("core") { sh "git config --global user.email \"test@zilliz.com\"" sh "git config --global user.name \"test\"" - // sh "./build.sh -l" + sh "./build.sh -l" withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -d /opt/milvus -j -u" } From 8cdd0a8b0ec2c640be8a219066594b7cacefe4c5 Mon Sep 17 00:00:00 2001 From: test Date: Wed, 16 Oct 2019 22:10:41 +0800 Subject: [PATCH 06/13] add build environment dockerfile Former-commit-id: 93add5191ee12ea5cfaed516d05a71039866279f --- docker/build_env/ubuntu16.04/Dockerfile | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) create mode 100644 docker/build_env/ubuntu16.04/Dockerfile diff --git a/docker/build_env/ubuntu16.04/Dockerfile b/docker/build_env/ubuntu16.04/Dockerfile new file mode 100644 index 0000000000..a0ccecce5f --- /dev/null +++ b/docker/build_env/ubuntu16.04/Dockerfile @@ -0,0 +1,25 @@ +FROM nvidia/cuda:10.1-devel-ubuntu16.04 + +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility + +RUN apt-get update && apt-get install -y --no-install-recommends wget && \ + wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ + wget -P /tmp https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ + apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ + sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \ + apt-get update && apt-get install -y --no-install-recommends \ + git flex bison gfortran \ + curl libtool automake libboost1.58-all-dev libssl-dev pkg-config libcurl4-openssl-dev \ + clang-format-6.0 clang-tidy-6.0 \ + lcov mysql-client libmysqlclient-dev intel-mkl-gnu-2019.4-243 intel-mkl-core-2019.4-243 && \ + apt-get remove --purge -y && \ + rm -rf /var/lib/apt/lists/* + +RUN ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so + +RUN sh -c 'echo export LD_LIBRARY_PATH=/opt/intel/compilers_and_libraries_2019.4.243/linux/mkl/lib/intel64:\$LD_LIBRARY_PATH > /etc/profile.d/mkl.sh' + +COPY docker-entrypoint.sh /app/docker-entrypoint.sh +ENTRYPOINT [ "/app/docker-entrypoint.sh" ] +CMD [ "start" ] + From 577b67dd16c4dd6ff4c0e7b603d8750f03c29e1a Mon Sep 17 00:00:00 2001 From: test Date: Wed, 16 Oct 2019 22:32:06 +0800 Subject: [PATCH 07/13] add milvus build environment docker-entrypoint.sh Former-commit-id: de3e29691876adfae48a6b55b5dc2e4d7ee99b3e --- docker/build_env/ubuntu16.04/docker-entrypoint.sh | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100755 docker/build_env/ubuntu16.04/docker-entrypoint.sh diff --git a/docker/build_env/ubuntu16.04/docker-entrypoint.sh b/docker/build_env/ubuntu16.04/docker-entrypoint.sh new file mode 100755 index 0000000000..1e85e7e9e1 --- /dev/null +++ b/docker/build_env/ubuntu16.04/docker-entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +if [ "$1" = 'start' ]; then + tail -f /dev/null +fi + +exec "$@" + From 6bce1f7a5727b139fce90d7b0585304e040ce216 Mon Sep 17 00:00:00 2001 From: test Date: Wed, 16 Oct 2019 22:41:24 +0800 Subject: [PATCH 08/13] update milvus-helm git url Former-commit-id: 6a71991be92332ca1cfdcf83e44d422b6544cd56 --- ci/jenkins/jenkinsfile/deploySingle2Dev.groovy | 2 +- ci/jenkins/jenkinsfile/singleDevTest.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/jenkins/jenkinsfile/deploySingle2Dev.groovy b/ci/jenkins/jenkinsfile/deploySingle2Dev.groovy index 87ded45255..f63da03b06 100644 --- a/ci/jenkins/jenkinsfile/deploySingle2Dev.groovy +++ b/ci/jenkins/jenkinsfile/deploySingle2Dev.groovy @@ -2,7 +2,7 @@ try { sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' sh 'helm repo update' dir ('milvus-helm') { - checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}", url: "https://github.com/zilliz-ci-robot/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]]) + checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}", url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]]) dir ("milvus-gpu") { sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELIEN_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/values.yaml --namespace milvus ." } diff --git a/ci/jenkins/jenkinsfile/singleDevTest.groovy b/ci/jenkins/jenkinsfile/singleDevTest.groovy index 1022dafd67..d228b34839 100644 --- a/ci/jenkins/jenkinsfile/singleDevTest.groovy +++ b/ci/jenkins/jenkinsfile/singleDevTest.groovy @@ -8,7 +8,7 @@ timeout(time: 30, unit: 'MINUTES') { if (!fileExists('milvus-helm')) { dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}", url: "https://github.com/zilliz-ci-robot/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]]) + checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_CREDENTIALS_ID}", url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]]) } } dir ("milvus-helm") { From 509a9a49d75bd23aeb1050407690e6c84a5687b1 Mon Sep 17 00:00:00 2001 From: test Date: Wed, 16 Oct 2019 22:56:35 +0800 Subject: [PATCH 09/13] update ci/jenkins/jenkinsfile/publishImages.groovy Former-commit-id: 4e2f7226865549b60a72d348d2e10bc1dcd4b5f4 --- ci/jenkins/jenkinsfile/publishImages.groovy | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/jenkins/jenkinsfile/publishImages.groovy b/ci/jenkins/jenkinsfile/publishImages.groovy index 5d19ab5a99..6c9ba14523 100644 --- a/ci/jenkins/jenkinsfile/publishImages.groovy +++ b/ci/jenkins/jenkinsfile/publishImages.groovy @@ -13,6 +13,10 @@ container('publish-images') { def imageName = "${PROJECT_NAME}/engine:${DOCKER_VERSION}" try { + def isExistImage = sh(returnStatus: true, script: "docker inspect --type=image ${imageName}") + if (isExistImage == 0) { + sh "docker rmi ${imageName}" + } def customImage = docker.build("${imageName}") docker.withRegistry('https://registry.zilliz.com', "${params.DOCKER_CREDENTIALS_ID}") { customImage.push() From d167310655364a144a4ada2cd07e84d8a4b625df Mon Sep 17 00:00:00 2001 From: test Date: Wed, 16 Oct 2019 23:07:48 +0800 Subject: [PATCH 10/13] update ci/jenkins/jenkinsfile/publishImages.groovy Former-commit-id: ed9ab1fd6cf8602965965459ec503233b4c682d4 --- ci/jenkins/jenkinsfile/publishImages.groovy | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/ci/jenkins/jenkinsfile/publishImages.groovy b/ci/jenkins/jenkinsfile/publishImages.groovy index 6c9ba14523..55864d64e7 100644 --- a/ci/jenkins/jenkinsfile/publishImages.groovy +++ b/ci/jenkins/jenkinsfile/publishImages.groovy @@ -2,6 +2,8 @@ container('publish-images') { timeout(time: 15, unit: 'MINUTES') { dir ("docker/deploy/${OS_NAME}") { def binaryPackage = "${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz" + def dockerRegistryURL = "registry.zilliz.com" + withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { def downloadStatus = sh(returnStatus: true, script: "curl -u${JFROG_USERNAME}:${JFROG_PASSWORD} -O ${params.JFROG_ARTFACTORY_URL}/milvus/package/${binaryPackage}") @@ -17,14 +19,29 @@ container('publish-images') { if (isExistImage == 0) { sh "docker rmi ${imageName}" } + def customImage = docker.build("${imageName}") - docker.withRegistry('https://registry.zilliz.com', "${params.DOCKER_CREDENTIALS_ID}") { + + isExistImage = sh(returnStatus: true, script: "docker inspect --type=image ${dockerRegistryURL}/${imageName}") + if (isExistImage == 0) { + sh "docker rmi ${dockerRegistryURL}/${imageName}" + } + + docker.withRegistry("${dockerRegistryURL}", "${params.DOCKER_CREDENTIALS_ID}") { customImage.push() } } catch (exc) { throw exc } finally { - sh "docker rmi ${imageName}" + def isExistImage = sh(returnStatus: true, script: "docker inspect --type=image ${imageName}") + if (isExistImage == 0) { + sh "docker rmi ${imageName}" + } + + isExistImage = sh(returnStatus: true, script: "docker inspect --type=image ${dockerRegistryURL}/${imageName}") + if (isExistImage == 0) { + sh "docker rmi ${dockerRegistryURL}/${imageName}" + } } } } From 90daa7a41d928d1afaa088786c0d07499e5f2ea0 Mon Sep 17 00:00:00 2001 From: test Date: Wed, 16 Oct 2019 23:23:49 +0800 Subject: [PATCH 11/13] fix docker registry url no protocol bug Former-commit-id: a1e199c86a8d9584e236e1ebccc2645d233c1420 --- ci/jenkins/jenkinsfile/publishImages.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/jenkins/jenkinsfile/publishImages.groovy b/ci/jenkins/jenkinsfile/publishImages.groovy index 55864d64e7..26db551206 100644 --- a/ci/jenkins/jenkinsfile/publishImages.groovy +++ b/ci/jenkins/jenkinsfile/publishImages.groovy @@ -27,7 +27,7 @@ container('publish-images') { sh "docker rmi ${dockerRegistryURL}/${imageName}" } - docker.withRegistry("${dockerRegistryURL}", "${params.DOCKER_CREDENTIALS_ID}") { + docker.withRegistry("https://${dockerRegistryURL}", "${params.DOCKER_CREDENTIALS_ID}") { customImage.push() } } catch (exc) { From f75012e0fac33d6c256e74b48c752bc26840c3d6 Mon Sep 17 00:00:00 2001 From: test Date: Wed, 16 Oct 2019 23:29:40 +0800 Subject: [PATCH 12/13] standard output is directed to /dev/null Former-commit-id: ffed4d2ffdc3ace17a3baee2e55802ec2174c54f --- ci/jenkins/jenkinsfile/publishImages.groovy | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/jenkins/jenkinsfile/publishImages.groovy b/ci/jenkins/jenkinsfile/publishImages.groovy index 26db551206..c6fbd922ff 100644 --- a/ci/jenkins/jenkinsfile/publishImages.groovy +++ b/ci/jenkins/jenkinsfile/publishImages.groovy @@ -15,14 +15,14 @@ container('publish-images') { def imageName = "${PROJECT_NAME}/engine:${DOCKER_VERSION}" try { - def isExistImage = sh(returnStatus: true, script: "docker inspect --type=image ${imageName}") + def isExistImage = sh(returnStatus: true, script: "docker inspect --type=image ${imageName} 2>&1 > /dev/null") if (isExistImage == 0) { sh "docker rmi ${imageName}" } def customImage = docker.build("${imageName}") - isExistImage = sh(returnStatus: true, script: "docker inspect --type=image ${dockerRegistryURL}/${imageName}") + isExistImage = sh(returnStatus: true, script: "docker inspect --type=image ${dockerRegistryURL}/${imageName} 2>&1 > /dev/null" if (isExistImage == 0) { sh "docker rmi ${dockerRegistryURL}/${imageName}" } @@ -33,12 +33,12 @@ container('publish-images') { } catch (exc) { throw exc } finally { - def isExistImage = sh(returnStatus: true, script: "docker inspect --type=image ${imageName}") + def isExistImage = sh(returnStatus: true, script: "docker inspect --type=image ${imageName} 2>&1 > /dev/null") if (isExistImage == 0) { sh "docker rmi ${imageName}" } - isExistImage = sh(returnStatus: true, script: "docker inspect --type=image ${dockerRegistryURL}/${imageName}") + isExistImage = sh(returnStatus: true, script: "docker inspect --type=image ${dockerRegistryURL}/${imageName} 2>&1 > /dev/null") if (isExistImage == 0) { sh "docker rmi ${dockerRegistryURL}/${imageName}" } From 2909cfcfd93184a44ba4aa2430c60aebb9037ecd Mon Sep 17 00:00:00 2001 From: test Date: Wed, 16 Oct 2019 23:42:57 +0800 Subject: [PATCH 13/13] fix publishImages.groovy bug Former-commit-id: 73b6ab5a4d28f7af93300dd5a281dc449f2dd756 --- ci/jenkins/jenkinsfile/publishImages.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/jenkins/jenkinsfile/publishImages.groovy b/ci/jenkins/jenkinsfile/publishImages.groovy index c6fbd922ff..1acaaaa700 100644 --- a/ci/jenkins/jenkinsfile/publishImages.groovy +++ b/ci/jenkins/jenkinsfile/publishImages.groovy @@ -22,7 +22,7 @@ container('publish-images') { def customImage = docker.build("${imageName}") - isExistImage = sh(returnStatus: true, script: "docker inspect --type=image ${dockerRegistryURL}/${imageName} 2>&1 > /dev/null" + isExistImage = sh(returnStatus: true, script: "docker inspect --type=image ${dockerRegistryURL}/${imageName} 2>&1 > /dev/null") if (isExistImage == 0) { sh "docker rmi ${dockerRegistryURL}/${imageName}" }