From bdc0239eba0d53fefbd3346e584b37aa20f03750 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Tue, 17 Dec 2019 19:15:17 +0800 Subject: [PATCH 01/13] update GRPC url --- core/cmake/ThirdPartyPackages.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/cmake/ThirdPartyPackages.cmake b/core/cmake/ThirdPartyPackages.cmake index 8d767f8200..0cca25ebba 100644 --- a/core/cmake/ThirdPartyPackages.cmake +++ b/core/cmake/ThirdPartyPackages.cmake @@ -331,7 +331,7 @@ if (DEFINED ENV{MILVUS_GRPC_URL}) set(GRPC_SOURCE_URL "$ENV{MILVUS_GRPC_URL}") else () set(GRPC_SOURCE_URL - "https://github.com/ZhifengZhang-CN/grpc-milvus/archive/${GRPC_VERSION}.zip") + "https://github.com/youny626/grpc-milvus/archive/${GRPC_VERSION}.zip") endif () set(GRPC_MD5 "0362ba219f59432c530070b5f5c3df73") From 865ca6ff133975b06856412e7636c88650a844ac Mon Sep 17 00:00:00 2001 From: quicksilver Date: Tue, 17 Dec 2019 19:35:26 +0800 Subject: [PATCH 02/13] debug codecov token secrets --- .github/workflows/core.yml | 4 ++-- docker-compose.yml | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index c1c09e01df..d2efab13e5 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -31,7 +31,7 @@ jobs: - name: Docker Build run: | docker-compose run --use-aliases -d db - docker-compose run ubuntu-core + docker-compose run -e MILVUS_CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} ubuntu-core - name: Docker Push if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' continue-on-error: true @@ -61,7 +61,7 @@ jobs: - name: Docker Build run: | docker-compose run --use-aliases -d db - docker-compose run centos-core + docker-compose run -e MILVUS_CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} centos-core - name: Docker Push if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' continue-on-error: true diff --git a/docker-compose.yml b/docker-compose.yml index 59b3e7ce46..a6e6e6e5a9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,6 +40,7 @@ services: - milvus command: &ubuntu-command > /bin/bash -c " + echo 'MILVUS_CODECOV_TOKEN: ${MILVUS_CODECOV_TOKEN}' /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql" @@ -60,6 +61,7 @@ services: - milvus command: ¢os-command > /bin/bash --login -c " + echo 'MILVUS_CODECOV_TOKEN: ${MILVUS_CODECOV_TOKEN}' /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql" From a2c857ccd8360d0b7b1ea89e4a5d377b681f601f Mon Sep 17 00:00:00 2001 From: quicksilver Date: Tue, 17 Dec 2019 19:37:04 +0800 Subject: [PATCH 03/13] debug codecov token secrets --- ci/jenkins/yaml/update_server_config.yaml | 2 -- 1 file changed, 2 deletions(-) delete mode 100644 ci/jenkins/yaml/update_server_config.yaml diff --git a/ci/jenkins/yaml/update_server_config.yaml b/ci/jenkins/yaml/update_server_config.yaml deleted file mode 100644 index 464365b577..0000000000 --- a/ci/jenkins/yaml/update_server_config.yaml +++ /dev/null @@ -1,2 +0,0 @@ -db_config: - primary_path: /var/lib/milvus From 186cab364f512a611665721e5f8f15d8f5f2cf79 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Tue, 17 Dec 2019 20:08:58 +0800 Subject: [PATCH 04/13] add codecov github actions --- .github/workflows/core.yml | 20 ++++++++++++++++++-- core/src/index/CMakeLists.txt | 4 ++-- docker-compose.yml | 2 -- 3 files changed, 20 insertions(+), 6 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index d2efab13e5..bab12631cc 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -31,7 +31,7 @@ jobs: - name: Docker Build run: | docker-compose run --use-aliases -d db - docker-compose run -e MILVUS_CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} ubuntu-core + docker-compose run ubuntu-core - name: Docker Push if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' continue-on-error: true @@ -40,6 +40,14 @@ jobs: docker login -u ${{ secrets.DOCKERHUB_USER }} \ -p ${{ secrets.DOCKERHUB_TOKEN }} docker-compose push ubuntu-core + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: ./ci/scripts/output_new.info + flags: unittests + yml: ./codecov.yaml + fail_ci_if_error: true centos: name: AMD64 CentOS ${{ matrix.centos }} @@ -61,7 +69,7 @@ jobs: - name: Docker Build run: | docker-compose run --use-aliases -d db - docker-compose run -e MILVUS_CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} centos-core + docker-compose run centos-core - name: Docker Push if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' continue-on-error: true @@ -70,3 +78,11 @@ jobs: docker login -u ${{ secrets.DOCKERHUB_USER }} \ -p ${{ secrets.DOCKERHUB_TOKEN }} docker-compose push centos-core + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: ./ci/scripts/output_new.info + flags: unittests + yml: ./codecov.yaml + fail_ci_if_error: true diff --git a/core/src/index/CMakeLists.txt b/core/src/index/CMakeLists.txt index 6e58566dd0..5664694155 100644 --- a/core/src/index/CMakeLists.txt +++ b/core/src/index/CMakeLists.txt @@ -98,9 +98,9 @@ if (CMAKE_BUILD_TYPE STREQUAL "Release") set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3") endif () else () - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -g -fPIC -DELPP_THREAD_SAFE -fopenmp -mavx -mf16c -msse4 -mpopcnt") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp -mavx -mf16c -msse4 -mpopcnt") if (KNOWHERE_GPU_VERSION) - set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3 -g") + set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") endif () endif () diff --git a/docker-compose.yml b/docker-compose.yml index a6e6e6e5a9..59b3e7ce46 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,7 +40,6 @@ services: - milvus command: &ubuntu-command > /bin/bash -c " - echo 'MILVUS_CODECOV_TOKEN: ${MILVUS_CODECOV_TOKEN}' /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql" @@ -61,7 +60,6 @@ services: - milvus command: ¢os-command > /bin/bash --login -c " - echo 'MILVUS_CODECOV_TOKEN: ${MILVUS_CODECOV_TOKEN}' /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql" From 75aee01d0d47d7ba2b9a3dc9196e90535b4ab1f5 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Tue, 17 Dec 2019 20:37:56 +0800 Subject: [PATCH 05/13] disable fail_ci_if_error option on codecov github actions --- .github/workflows/core.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index bab12631cc..a92fdb71a6 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -47,7 +47,6 @@ jobs: file: ./ci/scripts/output_new.info flags: unittests yml: ./codecov.yaml - fail_ci_if_error: true centos: name: AMD64 CentOS ${{ matrix.centos }} @@ -85,4 +84,3 @@ jobs: file: ./ci/scripts/output_new.info flags: unittests yml: ./codecov.yaml - fail_ci_if_error: true From c045d902bbe28d72c5ff0cc86b28d56704a578ad Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 18 Dec 2019 15:29:03 +0800 Subject: [PATCH 06/13] update check_ccache.sh --- ci/scripts/check_ccache.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/scripts/check_ccache.sh b/ci/scripts/check_ccache.sh index cc0708505e..c9c03ef149 100755 --- a/ci/scripts/check_ccache.sh +++ b/ci/scripts/check_ccache.sh @@ -44,7 +44,7 @@ check_ccache() { BRANCH=$1 PACKAGE_FILE="ccache-${OS_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" echo "fetching ${BRANCH}/${PACKAGE_FILE}" - wget -q --method HEAD "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" + wget -q --spider "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" if [[ $? == 0 ]];then wget -q "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" && \ mkdir -p ${CCACHE_DIRECTORY} && \ From c7f1c3ba213e8544fcb5169854056aaa1fc02b79 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 18 Dec 2019 15:54:13 +0800 Subject: [PATCH 07/13] update ci/jenkins/Jenkinsfile --- ci/jenkins/Jenkinsfile | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 12481eae12..9a05a714b7 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -107,6 +107,12 @@ pipeline { } } } + + post { + cleanup { + deleteDir() /* clean up our workspace */ + } + } } stage('Publish docker images') { @@ -129,6 +135,12 @@ pipeline { } } } + + post { + cleanup { + deleteDir() /* clean up our workspace */ + } + } } stage('Deploy to Development') { @@ -182,6 +194,7 @@ pipeline { } } } + post { unsuccessful { container('milvus-test-env') { @@ -190,6 +203,9 @@ pipeline { } } } + cleanup { + deleteDir() /* clean up our workspace */ + } } } } From 7dc5bf9fab31b0c24cc8a6dc32229637287fec0f Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 18 Dec 2019 15:59:50 +0800 Subject: [PATCH 08/13] update ci/jenkins/Jenkinsfile --- ci/jenkins/Jenkinsfile | 14 +------------- 1 file changed, 1 insertion(+), 13 deletions(-) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 9a05a714b7..8f2df91853 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -183,27 +183,15 @@ pipeline { } } } - - stage('Cleanup Dev') { - steps { - container('milvus-test-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" - } - } - } - } } post { - unsuccessful { + cleanup { container('milvus-test-env') { script { load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" } } - } - cleanup { deleteDir() /* clean up our workspace */ } } From 4e23e85cab6532155e4e1fa60aadeca454ccca59 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 18 Dec 2019 16:37:26 +0800 Subject: [PATCH 09/13] update ci/jenkins/Jenkinsfile --- ci/jenkins/Jenkinsfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 8f2df91853..f6303e03c5 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -66,7 +66,7 @@ pipeline { stage('Run Build') { agent { kubernetes { - label "${OS_NAME}-${BINRARY_VERSION}-build" + label "${OS_NAME}-${BINRARY_VERSION}-build-${env.BUILD_NUMBER}" defaultContainer 'jnlp' yamlFile "ci/jenkins/pod/milvus-${BINRARY_VERSION}-version-${OS_NAME}-build-env-pod.yaml" } @@ -118,7 +118,7 @@ pipeline { stage('Publish docker images') { agent { kubernetes { - label "${OS_NAME}-${BINRARY_VERSION}-publish" + label "${OS_NAME}-${BINRARY_VERSION}-publish-${env.BUILD_NUMBER}" defaultContainer 'jnlp' yamlFile 'ci/jenkins/pod/docker-pod.yaml' } @@ -152,7 +152,7 @@ pipeline { agent { kubernetes { - label "${OS_NAME}-${BINRARY_VERSION}-dev-test" + label "${OS_NAME}-${BINRARY_VERSION}-dev-test-${env.BUILD_NUMBER}" defaultContainer 'jnlp' yamlFile 'ci/jenkins/pod/testEnvironment.yaml' } From b2f4b21505250d96ef399bd7df183fc673223649 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 18 Dec 2019 17:09:00 +0800 Subject: [PATCH 10/13] update build.groovy --- ci/jenkins/step/build.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/jenkins/step/build.groovy b/ci/jenkins/step/build.groovy index 6bf5052400..d8b8f86f7b 100644 --- a/ci/jenkins/step/build.groovy +++ b/ci/jenkins/step/build.groovy @@ -4,7 +4,7 @@ timeout(time: 60, unit: 'MINUTES') { def checkResult = sh(script: "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache", returnStatus: true) if ("${BINRARY_VERSION}" == "gpu") { - sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -g -u -c\"" + sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -g -x -u -c\"" } else { sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -u -c\"" } From c90545c0f3a4219bb731746311f17fef7f796508 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 18 Dec 2019 17:19:15 +0800 Subject: [PATCH 11/13] update Jenkins CI --- ci/jenkins/Jenkinsfile | 24 +++++++++---------- ...vus-cpu-version-centos7-build-env-pod.yaml | 2 ++ ...cpu-version-ubuntu18.04-build-env-pod.yaml | 2 ++ ...vus-gpu-version-centos7-build-env-pod.yaml | 2 ++ ...gpu-version-ubuntu18.04-build-env-pod.yaml | 2 ++ ci/jenkins/step/build.groovy | 2 +- ci/jenkins/step/deploySingle2Dev.groovy | 2 +- ci/jenkins/step/publishImages.groovy | 2 +- ci/jenkins/step/singleDevNightlyTest.groovy | 2 +- ci/jenkins/step/singleDevTest.groovy | 2 +- ci/scripts/check_ccache.sh | 3 ++- ci/scripts/update_ccache.sh | 3 ++- 12 files changed, 29 insertions(+), 19 deletions(-) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index f6303e03c5..873c68ac02 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -48,7 +48,7 @@ pipeline { } axis { - name 'BINRARY_VERSION' + name 'BINARY_VERSION' values 'gpu', 'cpu' } } @@ -57,25 +57,25 @@ pipeline { stage("Milvus CI") { environment { PACKAGE_VERSION = VersionNumber([ - versionNumberString : '${SEMVER}-${BINRARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' + versionNumberString : '${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' ]); - DOCKER_VERSION = "${SEMVER}-${BINRARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}" + DOCKER_VERSION = "${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}" } stages { stage('Run Build') { agent { kubernetes { - label "${OS_NAME}-${BINRARY_VERSION}-build-${env.BUILD_NUMBER}" + label "${OS_NAME}-${BINARY_VERSION}-build-${env.BUILD_NUMBER}" defaultContainer 'jnlp' - yamlFile "ci/jenkins/pod/milvus-${BINRARY_VERSION}-version-${OS_NAME}-build-env-pod.yaml" + yamlFile "ci/jenkins/pod/milvus-${BINARY_VERSION}-version-${OS_NAME}-build-env-pod.yaml" } } stages { stage('Build') { steps { - container("milvus-${BINRARY_VERSION}-build-env") { + container("milvus-${BINARY_VERSION}-build-env") { script { load "${env.WORKSPACE}/ci/jenkins/step/build.groovy" } @@ -85,9 +85,9 @@ pipeline { stage('Code Coverage') { steps { - container("milvus-${BINRARY_VERSION}-build-env") { + container("milvus-${BINARY_VERSION}-build-env") { script { - if ("${BINRARY_VERSION}" == "gpu") { + if ("${BINARY_VERSION}" == "gpu") { load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" } else { echo "Skip Code Coverage" @@ -99,7 +99,7 @@ pipeline { stage('Upload Package') { steps { - container("milvus-${BINRARY_VERSION}-build-env") { + container("milvus-${BINARY_VERSION}-build-env") { script { load "${env.WORKSPACE}/ci/jenkins/step/package.groovy" } @@ -118,7 +118,7 @@ pipeline { stage('Publish docker images') { agent { kubernetes { - label "${OS_NAME}-${BINRARY_VERSION}-publish-${env.BUILD_NUMBER}" + label "${OS_NAME}-${BINARY_VERSION}-publish-${env.BUILD_NUMBER}" defaultContainer 'jnlp' yamlFile 'ci/jenkins/pod/docker-pod.yaml' } @@ -147,12 +147,12 @@ pipeline { environment { FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-") FORMAT_OS_NAME = "${OS_NAME}".replaceAll("\\.", "-") - HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${FORMAT_OS_NAME}-${BINRARY_VERSION}".toLowerCase() + HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${FORMAT_OS_NAME}-${BINARY_VERSION}".toLowerCase() } agent { kubernetes { - label "${OS_NAME}-${BINRARY_VERSION}-dev-test-${env.BUILD_NUMBER}" + label "${OS_NAME}-${BINARY_VERSION}-dev-test-${env.BUILD_NUMBER}" defaultContainer 'jnlp' yamlFile 'ci/jenkins/pod/testEnvironment.yaml' } diff --git a/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml b/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml index 6091a01eeb..f1628247b8 100644 --- a/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml @@ -16,6 +16,8 @@ spec: fieldPath: status.podIP - name: OS_NAME value: "centos7" + - name: BINARY_VERSION + value: "cpu" - name: BUILD_ENV_IMAGE_ID value: "225b4d9c26d67b70b476964b4dd6e216de4b464d7a973a8c0c7ed1313c4d81ad" command: diff --git a/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml b/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml index 4689da5db2..1c3fdc4bbf 100644 --- a/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml @@ -16,6 +16,8 @@ spec: fieldPath: status.podIP - name: OS_NAME value: "ubuntu18.04" + - name: BINARY_VERSION + value: "cpu" - name: BUILD_ENV_IMAGE_ID value: "23476391bec80c64f10d44a6370c73c71f011a6b95114b10ff82a60e771e11c7" command: diff --git a/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml b/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml index 34da55a93d..de3ce61630 100644 --- a/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml @@ -16,6 +16,8 @@ spec: fieldPath: status.podIP - name: OS_NAME value: "centos7" + - name: BINARY_VERSION + value: "gpu" - name: BUILD_ENV_IMAGE_ID value: "a5ec9914737ea4727d88ae36b4a73ca5d817f19438ba913cc1de6a1ee2ed2336" command: diff --git a/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml b/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml index 95b47c9111..8f941df63c 100644 --- a/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml @@ -16,6 +16,8 @@ spec: fieldPath: status.podIP - name: OS_NAME value: "ubuntu18.04" + - name: BINARY_VERSION + value: "gpu" - name: BUILD_ENV_IMAGE_ID value: "da9023b0f858f072672f86483a869aa87e90a5140864f89e5a012ec766d96dea" command: diff --git a/ci/jenkins/step/build.groovy b/ci/jenkins/step/build.groovy index d8b8f86f7b..2a12929575 100644 --- a/ci/jenkins/step/build.groovy +++ b/ci/jenkins/step/build.groovy @@ -3,7 +3,7 @@ timeout(time: 60, unit: 'MINUTES') { withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { def checkResult = sh(script: "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache", returnStatus: true) - if ("${BINRARY_VERSION}" == "gpu") { + if ("${BINARY_VERSION}" == "gpu") { sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -g -x -u -c\"" } else { sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -u -c\"" diff --git a/ci/jenkins/step/deploySingle2Dev.groovy b/ci/jenkins/step/deploySingle2Dev.groovy index e566c3da0c..e39dbcfb40 100644 --- a/ci/jenkins/step/deploySingle2Dev.groovy +++ b/ci/jenkins/step/deploySingle2Dev.groovy @@ -3,7 +3,7 @@ sh 'helm repo update' dir ('milvus-helm') { checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]]) dir ("milvus") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/sqlite_${BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/sqlite_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } diff --git a/ci/jenkins/step/publishImages.groovy b/ci/jenkins/step/publishImages.groovy index fe1c211abc..885fc5cf47 100644 --- a/ci/jenkins/step/publishImages.groovy +++ b/ci/jenkins/step/publishImages.groovy @@ -1,5 +1,5 @@ timeout(time: 15, unit: 'MINUTES') { - dir ("docker/deploy/${BINRARY_VERSION}/${OS_NAME}") { + dir ("docker/deploy/${BINARY_VERSION}/${OS_NAME}") { def binaryPackage = "${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz" withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { diff --git a/ci/jenkins/step/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy index 618c2457b3..0949041b01 100644 --- a/ci/jenkins/step/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -13,7 +13,7 @@ timeout(time: 90, unit: 'MINUTES') { } dir ("milvus-helm") { dir ("milvus") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } dir ("tests/milvus_python_test") { diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index 255f748996..279052eac1 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -14,7 +14,7 @@ timeout(time: 60, unit: 'MINUTES') { // } // dir ("milvus-helm") { // dir ("milvus") { - // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." // } // } // dir ("tests/milvus_python_test") { diff --git a/ci/scripts/check_ccache.sh b/ci/scripts/check_ccache.sh index c9c03ef149..98f95be7e2 100755 --- a/ci/scripts/check_ccache.sh +++ b/ci/scripts/check_ccache.sh @@ -1,6 +1,7 @@ #!/bin/bash OS_NAME="${OS_NAME}" +BINARY_VERSION="${BINARY_VERSION}" BUILD_ENV_DOCKER_IMAGE_ID="${BUILD_ENV_IMAGE_ID}" BRANCH_NAMES=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's=[a-zA-Z]*\/==g' | awk -F", " '{$1=""; print $0}') ARTIFACTORY_URL="" @@ -42,7 +43,7 @@ fi check_ccache() { BRANCH=$1 - PACKAGE_FILE="ccache-${OS_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" + PACKAGE_FILE="ccache-${OS_NAME}-${BINARY_VERSION}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" echo "fetching ${BRANCH}/${PACKAGE_FILE}" wget -q --spider "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" if [[ $? == 0 ]];then diff --git a/ci/scripts/update_ccache.sh b/ci/scripts/update_ccache.sh index ec52bd1156..605714fea5 100755 --- a/ci/scripts/update_ccache.sh +++ b/ci/scripts/update_ccache.sh @@ -1,6 +1,7 @@ #!/bin/bash OS_NAME="${OS_NAME}" +BINARY_VERSION="${BINARY_VERSION}" BUILD_ENV_DOCKER_IMAGE_ID="${BUILD_ENV_IMAGE_ID}" BRANCH_NAME=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's/.*, //' | sed 's=[a-zA-Z]*\/==g') ARTIFACTORY_URL="" @@ -50,7 +51,7 @@ if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then exit 1 fi -PACKAGE_FILE="ccache-${OS_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" +PACKAGE_FILE="ccache-${OS_NAME}-${BINARY_VERSION}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" REMOTE_PACKAGE_PATH="${ARTIFACTORY_URL}/${BRANCH_NAME}" ccache --show-stats From 3e8e70115b7cc9a9b9c0703db9f109abcb4ccca5 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 18 Dec 2019 17:22:12 +0800 Subject: [PATCH 12/13] update Jenkins CI --- ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml | 2 -- .../pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml | 2 -- ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml | 2 -- .../pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml | 2 -- ci/scripts/check_ccache.sh | 3 +-- ci/scripts/update_ccache.sh | 3 +-- 6 files changed, 2 insertions(+), 12 deletions(-) diff --git a/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml b/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml index f1628247b8..6091a01eeb 100644 --- a/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml @@ -16,8 +16,6 @@ spec: fieldPath: status.podIP - name: OS_NAME value: "centos7" - - name: BINARY_VERSION - value: "cpu" - name: BUILD_ENV_IMAGE_ID value: "225b4d9c26d67b70b476964b4dd6e216de4b464d7a973a8c0c7ed1313c4d81ad" command: diff --git a/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml b/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml index 1c3fdc4bbf..4689da5db2 100644 --- a/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml @@ -16,8 +16,6 @@ spec: fieldPath: status.podIP - name: OS_NAME value: "ubuntu18.04" - - name: BINARY_VERSION - value: "cpu" - name: BUILD_ENV_IMAGE_ID value: "23476391bec80c64f10d44a6370c73c71f011a6b95114b10ff82a60e771e11c7" command: diff --git a/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml b/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml index de3ce61630..34da55a93d 100644 --- a/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml @@ -16,8 +16,6 @@ spec: fieldPath: status.podIP - name: OS_NAME value: "centos7" - - name: BINARY_VERSION - value: "gpu" - name: BUILD_ENV_IMAGE_ID value: "a5ec9914737ea4727d88ae36b4a73ca5d817f19438ba913cc1de6a1ee2ed2336" command: diff --git a/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml b/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml index 8f941df63c..95b47c9111 100644 --- a/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml @@ -16,8 +16,6 @@ spec: fieldPath: status.podIP - name: OS_NAME value: "ubuntu18.04" - - name: BINARY_VERSION - value: "gpu" - name: BUILD_ENV_IMAGE_ID value: "da9023b0f858f072672f86483a869aa87e90a5140864f89e5a012ec766d96dea" command: diff --git a/ci/scripts/check_ccache.sh b/ci/scripts/check_ccache.sh index 98f95be7e2..c9c03ef149 100755 --- a/ci/scripts/check_ccache.sh +++ b/ci/scripts/check_ccache.sh @@ -1,7 +1,6 @@ #!/bin/bash OS_NAME="${OS_NAME}" -BINARY_VERSION="${BINARY_VERSION}" BUILD_ENV_DOCKER_IMAGE_ID="${BUILD_ENV_IMAGE_ID}" BRANCH_NAMES=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's=[a-zA-Z]*\/==g' | awk -F", " '{$1=""; print $0}') ARTIFACTORY_URL="" @@ -43,7 +42,7 @@ fi check_ccache() { BRANCH=$1 - PACKAGE_FILE="ccache-${OS_NAME}-${BINARY_VERSION}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" + PACKAGE_FILE="ccache-${OS_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" echo "fetching ${BRANCH}/${PACKAGE_FILE}" wget -q --spider "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" if [[ $? == 0 ]];then diff --git a/ci/scripts/update_ccache.sh b/ci/scripts/update_ccache.sh index 605714fea5..ec52bd1156 100755 --- a/ci/scripts/update_ccache.sh +++ b/ci/scripts/update_ccache.sh @@ -1,7 +1,6 @@ #!/bin/bash OS_NAME="${OS_NAME}" -BINARY_VERSION="${BINARY_VERSION}" BUILD_ENV_DOCKER_IMAGE_ID="${BUILD_ENV_IMAGE_ID}" BRANCH_NAME=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's/.*, //' | sed 's=[a-zA-Z]*\/==g') ARTIFACTORY_URL="" @@ -51,7 +50,7 @@ if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then exit 1 fi -PACKAGE_FILE="ccache-${OS_NAME}-${BINARY_VERSION}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" +PACKAGE_FILE="ccache-${OS_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" REMOTE_PACKAGE_PATH="${ARTIFACTORY_URL}/${BRANCH_NAME}" ccache --show-stats From 42342d01cb12c0115b9687f3209fb96ca1f5e22c Mon Sep 17 00:00:00 2001 From: quicksilver Date: Thu, 19 Dec 2019 10:41:42 +0800 Subject: [PATCH 13/13] update Jenkins CI --- ci/jenkins/Jenkinsfile | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 873c68ac02..aed986e012 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -107,12 +107,6 @@ pipeline { } } } - - post { - cleanup { - deleteDir() /* clean up our workspace */ - } - } } stage('Publish docker images') { @@ -135,12 +129,6 @@ pipeline { } } } - - post { - cleanup { - deleteDir() /* clean up our workspace */ - } - } } stage('Deploy to Development') { @@ -192,7 +180,6 @@ pipeline { load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" } } - deleteDir() /* clean up our workspace */ } } }