From 20aee0309398f8d82b92b9e8b05e4d536371916e Mon Sep 17 00:00:00 2001 From: quicksilver Date: Thu, 19 Dec 2019 15:31:47 +0800 Subject: [PATCH 1/8] add test enviroment images --- docker/test_env/Dockerfile | 14 ++++++++++++++ docker/test_env/docker-entrypoint.sh | 9 +++++++++ 2 files changed, 23 insertions(+) create mode 100644 docker/test_env/Dockerfile create mode 100755 docker/test_env/docker-entrypoint.sh diff --git a/docker/test_env/Dockerfile b/docker/test_env/Dockerfile new file mode 100644 index 0000000000..0699b5a642 --- /dev/null +++ b/docker/test_env/Dockerfile @@ -0,0 +1,14 @@ +FROM python:3.6.8-jessie + +RUN apt-get update && apt-get install -y --no-install-recommends wget apt-transport-https && \ + wget -qO- "https://get.helm.sh/helm-v3.0.2-linux-amd64.tar.gz" | tar --strip-components=1 -xz -C /usr/local/bin linux-amd64/helm && \ + sh -c 'echo deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main > /etc/apt/sources.list.d/kubernetes.list' && \ + apt-get update && apt-get install -y --no-install-recommends \ + build-essential kubectl && \ + apt-get remove --purge -y + +COPY docker-entrypoint.sh /app/docker-entrypoint.sh +WORKDIR /root + +ENTRYPOINT [ "/app/docker-entrypoint.sh" ] +CMD [ "start" ] diff --git a/docker/test_env/docker-entrypoint.sh b/docker/test_env/docker-entrypoint.sh new file mode 100755 index 0000000000..af9ba0ba66 --- /dev/null +++ b/docker/test_env/docker-entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e + +if [ "$1" = 'start' ]; then + tail -f /dev/null +fi + +exec "$@" \ No newline at end of file From 60eb5a0826122a2b1371fc1d0aa35df426e98d1a Mon Sep 17 00:00:00 2001 From: zilliz Date: Thu, 19 Dec 2019 15:49:48 +0800 Subject: [PATCH 2/8] update test enviroment images Dockerfile --- docker/test_env/Dockerfile | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/test_env/Dockerfile b/docker/test_env/Dockerfile index 0699b5a642..ba8ac8b87c 100644 --- a/docker/test_env/Dockerfile +++ b/docker/test_env/Dockerfile @@ -2,6 +2,8 @@ FROM python:3.6.8-jessie RUN apt-get update && apt-get install -y --no-install-recommends wget apt-transport-https && \ wget -qO- "https://get.helm.sh/helm-v3.0.2-linux-amd64.tar.gz" | tar --strip-components=1 -xz -C /usr/local/bin linux-amd64/helm && \ + wget -P /tmp https://mirrors.aliyun.com/kubernetes/apt/doc/apt-key.gpg && \ + apt-key add /tmp/apt-key.gpg && \ sh -c 'echo deb https://mirrors.aliyun.com/kubernetes/apt/ kubernetes-xenial main > /etc/apt/sources.list.d/kubernetes.list' && \ apt-get update && apt-get install -y --no-install-recommends \ build-essential kubectl && \ From 8c64d4c042747360f583fa3099b4496c6b9d75ee Mon Sep 17 00:00:00 2001 From: zilliz Date: Thu, 19 Dec 2019 16:07:26 +0800 Subject: [PATCH 3/8] Use Helm version 3 on Jenkins CI --- ci/jenkins/pod/testEnvironment.yaml | 2 +- ci/jenkins/step/cleanupSingleDev.groovy | 8 ++++---- ci/jenkins/step/deploySingle2Dev.groovy | 4 ++-- ci/jenkins/step/singleDevNightlyTest.groovy | 2 +- ci/jenkins/step/singleDevTest.groovy | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/ci/jenkins/pod/testEnvironment.yaml b/ci/jenkins/pod/testEnvironment.yaml index 174277dfcc..3f6a17c2a9 100644 --- a/ci/jenkins/pod/testEnvironment.yaml +++ b/ci/jenkins/pod/testEnvironment.yaml @@ -7,7 +7,7 @@ metadata: spec: containers: - name: milvus-test-env - image: registry.zilliz.com/milvus/milvus-test-env:v0.1 + image: registry.zilliz.com/milvus/milvus-test-env:v0.2 command: - cat tty: true diff --git a/ci/jenkins/step/cleanupSingleDev.groovy b/ci/jenkins/step/cleanupSingleDev.groovy index 101105c027..171da361f7 100644 --- a/ci/jenkins/step/cleanupSingleDev.groovy +++ b/ci/jenkins/step/cleanupSingleDev.groovy @@ -1,12 +1,12 @@ try { - def helmResult = sh script: "helm status ${env.HELM_RELEASE_NAME}", returnStatus: true + def helmResult = sh script: "helm status -n milvus ${env.HELM_RELEASE_NAME}", returnStatus: true if (!helmResult) { - sh "helm del --purge ${env.HELM_RELEASE_NAME}" + sh "helm uninstall -n milvus ${env.HELM_RELEASE_NAME}" } } catch (exc) { - def helmResult = sh script: "helm status ${env.HELM_RELEASE_NAME}", returnStatus: true + def helmResult = sh script: "helm status -n milvus ${env.HELM_RELEASE_NAME}", returnStatus: true if (!helmResult) { - sh "helm del --purge ${env.HELM_RELEASE_NAME}" + sh "helm uninstall -n milvus ${env.HELM_RELEASE_NAME}" } throw exc } diff --git a/ci/jenkins/step/deploySingle2Dev.groovy b/ci/jenkins/step/deploySingle2Dev.groovy index e39dbcfb40..914e9605bd 100644 --- a/ci/jenkins/step/deploySingle2Dev.groovy +++ b/ci/jenkins/step/deploySingle2Dev.groovy @@ -1,9 +1,9 @@ -sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' +sh 'helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' sh 'helm repo update' dir ('milvus-helm') { checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]]) dir ("milvus") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/sqlite_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/db_backend/sqlite_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." } } diff --git a/ci/jenkins/step/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy index 0949041b01..c980127028 100644 --- a/ci/jenkins/step/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -13,7 +13,7 @@ timeout(time: 90, unit: 'MINUTES') { } dir ("milvus-helm") { dir ("milvus") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." } } dir ("tests/milvus_python_test") { diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index 279052eac1..9d60ea48bc 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -14,7 +14,7 @@ timeout(time: 60, unit: 'MINUTES') { // } // dir ("milvus-helm") { // dir ("milvus") { - // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." // } // } // dir ("tests/milvus_python_test") { From 549068eb48fad1dc00c4fe78fcdf74787f44087b Mon Sep 17 00:00:00 2001 From: zilliz Date: Thu, 19 Dec 2019 17:01:34 +0800 Subject: [PATCH 4/8] Use Helm version 3 on Jenkins CI --- ci/jenkins/step/deploySingle2Dev.groovy | 2 +- ci/jenkins/step/singleDevNightlyTest.groovy | 2 +- ci/jenkins/step/singleDevTest.groovy | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/jenkins/step/deploySingle2Dev.groovy b/ci/jenkins/step/deploySingle2Dev.groovy index 914e9605bd..db8f701fb3 100644 --- a/ci/jenkins/step/deploySingle2Dev.groovy +++ b/ci/jenkins/step/deploySingle2Dev.groovy @@ -3,7 +3,7 @@ sh 'helm repo update' dir ('milvus-helm') { checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]]) dir ("milvus") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/db_backend/sqlite_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." + sh "helm install --wait --timeout --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/db_backend/sqlite_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." } } diff --git a/ci/jenkins/step/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy index c980127028..78e9003e85 100644 --- a/ci/jenkins/step/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -13,7 +13,7 @@ timeout(time: 90, unit: 'MINUTES') { } dir ("milvus-helm") { dir ("milvus") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." + sh "helm install --wait --timeout --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." } } dir ("tests/milvus_python_test") { diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index 9d60ea48bc..3c4a577a61 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -14,7 +14,7 @@ timeout(time: 60, unit: 'MINUTES') { // } // dir ("milvus-helm") { // dir ("milvus") { - // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." + // sh "helm install --wait --timeout --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." // } // } // dir ("tests/milvus_python_test") { From 4a68f9e52afa4687932b3f84a2cc2265f3baa403 Mon Sep 17 00:00:00 2001 From: zilliz Date: Thu, 19 Dec 2019 17:37:28 +0800 Subject: [PATCH 5/8] Use Helm version 3 on Jenkins CI --- ci/jenkins/step/deploySingle2Dev.groovy | 2 +- ci/jenkins/step/singleDevNightlyTest.groovy | 2 +- ci/jenkins/step/singleDevTest.groovy | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/jenkins/step/deploySingle2Dev.groovy b/ci/jenkins/step/deploySingle2Dev.groovy index db8f701fb3..edb1c62a5f 100644 --- a/ci/jenkins/step/deploySingle2Dev.groovy +++ b/ci/jenkins/step/deploySingle2Dev.groovy @@ -3,7 +3,7 @@ sh 'helm repo update' dir ('milvus-helm') { checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]]) dir ("milvus") { - sh "helm install --wait --timeout --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/db_backend/sqlite_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." + sh "helm install --wait --timeout 300s --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/db_backend/sqlite_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." } } diff --git a/ci/jenkins/step/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy index 78e9003e85..74a8768cba 100644 --- a/ci/jenkins/step/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -13,7 +13,7 @@ timeout(time: 90, unit: 'MINUTES') { } dir ("milvus-helm") { dir ("milvus") { - sh "helm install --wait --timeout --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." + sh "helm install --wait --timeout 300s --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." } } dir ("tests/milvus_python_test") { diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index 3c4a577a61..291b4470e5 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -14,7 +14,7 @@ timeout(time: 60, unit: 'MINUTES') { // } // dir ("milvus-helm") { // dir ("milvus") { - // sh "helm install --wait --timeout --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." + // sh "helm install --wait --timeout 300s --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ${env.HELM_RELEASE_NAME} ." // } // } // dir ("tests/milvus_python_test") { From 1ab333ba1aa8a046c6867a51e4bc019660445712 Mon Sep 17 00:00:00 2001 From: zilliz Date: Thu, 19 Dec 2019 18:52:08 +0800 Subject: [PATCH 6/8] Use Helm version 3 on Jenkins CI --- ci/jenkins/Jenkinsfile | 6 +++--- ci/jenkins/step/deploySingle2Dev.groovy | 1 + 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index aed986e012..9c647146a6 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -66,7 +66,7 @@ pipeline { stage('Run Build') { agent { kubernetes { - label "${OS_NAME}-${BINARY_VERSION}-build-${env.BUILD_NUMBER}" + label "${OS_NAME}-${BINARY_VERSION}-build-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}" defaultContainer 'jnlp' yamlFile "ci/jenkins/pod/milvus-${BINARY_VERSION}-version-${OS_NAME}-build-env-pod.yaml" } @@ -112,7 +112,7 @@ pipeline { stage('Publish docker images') { agent { kubernetes { - label "${OS_NAME}-${BINARY_VERSION}-publish-${env.BUILD_NUMBER}" + label "${OS_NAME}-${BINARY_VERSION}-publish-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}" defaultContainer 'jnlp' yamlFile 'ci/jenkins/pod/docker-pod.yaml' } @@ -140,7 +140,7 @@ pipeline { agent { kubernetes { - label "${OS_NAME}-${BINARY_VERSION}-dev-test-${env.BUILD_NUMBER}" + label "${OS_NAME}-${BINARY_VERSION}-dev-test-${env.PIPELINE_NAME}-${env.BUILD_NUMBER}" defaultContainer 'jnlp' yamlFile 'ci/jenkins/pod/testEnvironment.yaml' } diff --git a/ci/jenkins/step/deploySingle2Dev.groovy b/ci/jenkins/step/deploySingle2Dev.groovy index edb1c62a5f..3a5854b58d 100644 --- a/ci/jenkins/step/deploySingle2Dev.groovy +++ b/ci/jenkins/step/deploySingle2Dev.groovy @@ -1,3 +1,4 @@ +sh 'helm version' sh 'helm repo add stable https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' sh 'helm repo update' dir ('milvus-helm') { From 02ff4d8a5b00febf30cd90711c4f618fe7db2fb8 Mon Sep 17 00:00:00 2001 From: zilliz Date: Fri, 20 Dec 2019 16:12:23 +0800 Subject: [PATCH 7/8] Add Test Framework resource limit on Jenkins CI --- ci/jenkins/pod/testEnvironment.yaml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ci/jenkins/pod/testEnvironment.yaml b/ci/jenkins/pod/testEnvironment.yaml index 3f6a17c2a9..da1af44992 100644 --- a/ci/jenkins/pod/testEnvironment.yaml +++ b/ci/jenkins/pod/testEnvironment.yaml @@ -11,6 +11,13 @@ spec: command: - cat tty: true + resources: + limits: + memory: "8Gi" + cpu: "4.0" + requests: + memory: "4Gi" + cpu: "2.0" volumeMounts: - name: kubeconf mountPath: /root/.kube/ From 90a800cbee307bd2bd988bbfae37370d2f9fee17 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sat, 21 Dec 2019 10:38:12 +0800 Subject: [PATCH 8/8] update pods resource limite on Jenkins CI --- ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml | 2 +- .../pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml | 2 +- ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml | 2 +- .../pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml b/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml index 6091a01eeb..775e9f628f 100644 --- a/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml @@ -24,7 +24,7 @@ spec: resources: limits: memory: "12Gi" - cpu: "8.0" + cpu: "4.0" requests: memory: "8Gi" cpu: "4.0" diff --git a/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml b/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml index 4689da5db2..f36569407a 100644 --- a/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml @@ -24,7 +24,7 @@ spec: resources: limits: memory: "12Gi" - cpu: "8.0" + cpu: "4.0" requests: memory: "8Gi" cpu: "4.0" diff --git a/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml b/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml index 34da55a93d..2164db267c 100644 --- a/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml @@ -24,7 +24,7 @@ spec: resources: limits: memory: "12Gi" - cpu: "8.0" + cpu: "4.0" nvidia.com/gpu: 1 requests: memory: "8Gi" diff --git a/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml b/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml index 95b47c9111..43a4152342 100644 --- a/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml @@ -24,7 +24,7 @@ spec: resources: limits: memory: "12Gi" - cpu: "8.0" + cpu: "4.0" nvidia.com/gpu: 1 requests: memory: "8Gi"