From 5c4889dd257e1ef95d1d455712d474ef434468fa Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 25 Sep 2019 15:04:42 +0800 Subject: [PATCH 1/5] format CMakeLists.txt Former-commit-id: 447d58d1b8d0a065f2c5c5ec7b7aa3b57babe8b3 --- ci/jenkinsfile/cluster_deploy2dev.groovy | 2 +- ci/jenkinsfile/deploy2dev.groovy | 2 +- ci/jenkinsfile/deploy2staging.groovy | 2 +- ci/jenkinsfile/dev_test.groovy | 2 +- ci/jenkinsfile/staging_test.groovy | 2 +- cpp/CMakeLists.txt | 2 +- cpp/cmake/ThirdPartyPackages.cmake | 31 +++++++++++++++--------- 7 files changed, 26 insertions(+), 17 deletions(-) diff --git a/ci/jenkinsfile/cluster_deploy2dev.groovy b/ci/jenkinsfile/cluster_deploy2dev.groovy index 7c7a94999d..7f2a584256 100644 --- a/ci/jenkinsfile/cluster_deploy2dev.groovy +++ b/ci/jenkinsfile/cluster_deploy2dev.groovy @@ -5,7 +5,7 @@ try { dir ("milvus-helm") { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) dir ("milvus/milvus-cluster") { - sh "helm install --wait --timeout 300 --set roServers.image.tag=${DOCKER_VERSION} --set woServers.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/values.yaml --name ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster --namespace milvus-cluster --version 0.4.0 . " + sh "helm install --wait --timeout 300 --set roServers.image.tag=${DOCKER_VERSION} --set woServers.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP -f ci/values.yaml --name ${env.JOB_NAME}-${env.BUILD_NUMBER}-cluster --namespace milvus-cluster --version 0.5.0 . " } } /* diff --git a/ci/jenkinsfile/deploy2dev.groovy b/ci/jenkinsfile/deploy2dev.groovy index 8278b7745e..b00c8fa335 100644 --- a/ci/jenkinsfile/deploy2dev.groovy +++ b/ci/jenkinsfile/deploy2dev.groovy @@ -5,7 +5,7 @@ try { dir ("milvus-helm") { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.4.0 ." + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.5.0 ." } } } catch (exc) { diff --git a/ci/jenkinsfile/deploy2staging.groovy b/ci/jenkinsfile/deploy2staging.groovy index ee01097de8..42ccfda71a 100644 --- a/ci/jenkinsfile/deploy2staging.groovy +++ b/ci/jenkinsfile/deploy2staging.groovy @@ -5,7 +5,7 @@ try { dir ("milvus-helm") { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus-helm.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.repository=\"zilliz.azurecr.cn/milvus/engine\" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.4.0 ." + sh "helm install --wait --timeout 300 --set engine.image.repository=\"zilliz.azurecr.cn/milvus/engine\" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/values.yaml --namespace milvus-1 --version 0.5.0 ." } } } catch (exc) { diff --git a/ci/jenkinsfile/dev_test.groovy b/ci/jenkinsfile/dev_test.groovy index 623e4092cd..cf8d9b66b0 100644 --- a/ci/jenkinsfile/dev_test.groovy +++ b/ci/jenkinsfile/dev_test.groovy @@ -16,7 +16,7 @@ timeout(time: 30, unit: 'MINUTES') { } dir ("milvus-helm") { dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.4.0 ." + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.5.0 ." } } dir ("${PROJECT_NAME}_test") { diff --git a/ci/jenkinsfile/staging_test.groovy b/ci/jenkinsfile/staging_test.groovy index 7fceee5dfd..dcf1787103 100644 --- a/ci/jenkinsfile/staging_test.groovy +++ b/ci/jenkinsfile/staging_test.groovy @@ -17,7 +17,7 @@ timeout(time: 40, unit: 'MINUTES') { } dir ("milvus-helm") { dir ("milvus/milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.repository=\"zilliz.azurecr.cn/milvus/engine\" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.4.0 ." + sh "helm install --wait --timeout 300 --set engine.image.repository=\"zilliz.azurecr.cn/milvus/engine\" --set engine.image.tag=${DOCKER_VERSION} --set expose.type=loadBalancer --name ${env.JOB_NAME}-${env.BUILD_NUMBER} -f ci/db_backend/mysql_values.yaml --namespace milvus-2 --version 0.5.0 ." } } dir ("${PROJECT_NAME}_test") { diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt index e7cf83d704..1b5c4fc161 100644 --- a/cpp/CMakeLists.txt +++ b/cpp/CMakeLists.txt @@ -71,7 +71,7 @@ if(MILVUS_VERSION_MAJOR STREQUAL "" OR MILVUS_VERSION_MINOR STREQUAL "" OR MILVUS_VERSION_PATCH STREQUAL "") message(WARNING "Failed to determine Milvus version from git branch name") - set(MILVUS_VERSION "0.4.0") + set(MILVUS_VERSION "0.5.0") endif() message(STATUS "Build version = ${MILVUS_VERSION}") diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index f280c48293..37e9f4c3cd 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -137,9 +137,26 @@ if(NOT DEFINED USE_JFROG_CACHE) set(USE_JFROG_CACHE "OFF") endif() if(USE_JFROG_CACHE STREQUAL "ON") - set(JFROG_ARTFACTORY_CACHE_URL "http://192.168.1.201:80/artifactory/generic-local/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}") - set(JFROG_USER_NAME "test") - set(JFROG_PASSWORD "Fantast1c") + if(DEFINED ENV{JFROG_ARTFACTORY_URL}) + set(JFROG_ARTFACTORY_URL "$ENV{JFROG_ARTFACTORY_URL}") + endif() + if(NOT DEFINED JFROG_ARTFACTORY_URL) + message(FATAL_ERROR "JFROG_ARTFACTORY_URL is not set") + endif() + set(JFROG_ARTFACTORY_CACHE_URL "{JFROG_ARTFACTORY_URL}/generic-local/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}") + if(DEFINED ENV{JFROG_USER_NAME}) + set(JFROG_USER_NAME "$ENV{JFROG_USER_NAME}") + endif() + if(NOT DEFINED JFROG_USER_NAME) + message(FATAL_ERROR "JFROG_USER_NAME is not set") + endif() + if(DEFINED ENV{JFROG_PASSWORD}) + set(JFROG_PASSWORD "$ENV{JFROG_PASSWORD}") + endif() + if(NOT DEFINED JFROG_PASSWORD) + message(FATAL_ERROR "JFROG_PASSWORD is not set") + endif() + set(THIRDPARTY_PACKAGE_CACHE "${THIRDPARTY_DIR}/cache") endif() @@ -242,14 +259,6 @@ foreach(_VERSION_ENTRY ${TOOLCHAIN_VERSIONS_TXT}) set(${_LIB_NAME} "${_LIB_VERSION}") endforeach() -if(DEFINED ENV{MILVUS_ARROW_URL}) - set(ARROW_SOURCE_URL "$ENV{MILVUS_ARROW_URL}") -else() - set(ARROW_SOURCE_URL - "https://github.com/youny626/arrow.git" - ) -endif() - if(DEFINED ENV{MILVUS_BOOST_URL}) set(BOOST_SOURCE_URL "$ENV{MILVUS_BOOST_URL}") else() From 41a2cc343254045d1764b19edd231599ed97ba46 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 25 Sep 2019 15:17:06 +0800 Subject: [PATCH 2/5] format CMakeLists.txt Former-commit-id: 281e0bcb78233f6570c426fab6f9a4a067adf790 --- cpp/cmake/ThirdPartyPackages.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 37e9f4c3cd..a178468b0f 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -143,7 +143,7 @@ if(USE_JFROG_CACHE STREQUAL "ON") if(NOT DEFINED JFROG_ARTFACTORY_URL) message(FATAL_ERROR "JFROG_ARTFACTORY_URL is not set") endif() - set(JFROG_ARTFACTORY_CACHE_URL "{JFROG_ARTFACTORY_URL}/generic-local/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}") + set(JFROG_ARTFACTORY_CACHE_URL "${JFROG_ARTFACTORY_URL}/generic-local/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${MILVUS_BUILD_ARCH}/${BUILD_TYPE}") if(DEFINED ENV{JFROG_USER_NAME}) set(JFROG_USER_NAME "$ENV{JFROG_USER_NAME}") endif() From 0f287c39100d50c38a9db5135f5c3d7373677f5b Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 25 Sep 2019 16:31:30 +0800 Subject: [PATCH 3/5] format CMakeLists.txt Former-commit-id: b1980e2e8fda99a49d9bd005ce0caf0ef494ca02 --- cpp/src/core/cmake/ThirdPartyPackagesCore.cmake | 3 --- 1 file changed, 3 deletions(-) diff --git a/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake b/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake index fad0869021..7b0c75633d 100644 --- a/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake +++ b/cpp/src/core/cmake/ThirdPartyPackagesCore.cmake @@ -123,9 +123,6 @@ if(NOT DEFINED USE_JFROG_CACHE) set(USE_JFROG_CACHE "OFF") endif() if(USE_JFROG_CACHE STREQUAL "ON") - set(JFROG_ARTFACTORY_CACHE_URL "http://192.168.1.201:80/artifactory/generic-local/milvus/thirdparty/cache/${CMAKE_OS_NAME}/${KNOWHERE_BUILD_ARCH}/${BUILD_TYPE}") - set(JFROG_USER_NAME "test") - set(JFROG_PASSWORD "Fantast1c") set(THIRDPARTY_PACKAGE_CACHE "${THIRDPARTY_DIR}/cache") endif() From e9307c3c240aa16610c77717bed361c8d8a81618 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 25 Sep 2019 17:30:14 +0800 Subject: [PATCH 4/5] add JFROG_USER parameter Former-commit-id: 7bb920f6efc3f33649f0766b34e1878872c75039 --- ci/jenkinsfile/milvus_build.groovy | 11 +++-------- ci/jenkinsfile/milvus_build_no_ut.groovy | 11 +++-------- 2 files changed, 6 insertions(+), 16 deletions(-) diff --git a/ci/jenkinsfile/milvus_build.groovy b/ci/jenkinsfile/milvus_build.groovy index 0e24c63a34..289b9a6fe8 100644 --- a/ci/jenkinsfile/milvus_build.groovy +++ b/ci/jenkinsfile/milvus_build.groovy @@ -5,17 +5,12 @@ container('milvus-build-env') { try { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - /* - dir ("cpp/thirdparty/knowhere") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/knowhere.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - sh "./build.sh -t ${params.BUILD_TYPE} -p ${knowhere_build_dir} -j" - } - */ - dir ("cpp") { sh "git config --global user.email \"test@zilliz.com\"" sh "git config --global user.name \"test\"" - sh "./build.sh -t ${params.BUILD_TYPE} -j -u -c" + withCredentials([usernamePassword(credentialsId: "${params.JFROG_USER}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { + sh "export JFROG_ARTFACTORY_URL='http://192.168.1.201:80/artifactory' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -j -u -c" + } } } catch (exc) { updateGitlabCommitStatus name: 'Build Engine', state: 'failed' diff --git a/ci/jenkinsfile/milvus_build_no_ut.groovy b/ci/jenkinsfile/milvus_build_no_ut.groovy index dd0fe4a442..4caa9f0fa7 100644 --- a/ci/jenkinsfile/milvus_build_no_ut.groovy +++ b/ci/jenkinsfile/milvus_build_no_ut.groovy @@ -5,17 +5,12 @@ container('milvus-build-env') { try { checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/milvus.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - /* - dir ("cpp/thirdparty/knowhere") { - checkout([$class: 'GitSCM', branches: [[name: "${SEMVER}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'SubmoduleOption',disableSubmodules: false,parentCredentials: true,recursiveSubmodules: true,reference: '',trackingSubmodules: false]], submoduleCfg: [], userRemoteConfigs: [[credentialsId: "${params.GIT_USER}", url: "git@192.168.1.105:megasearch/knowhere.git", name: 'origin', refspec: "+refs/heads/${SEMVER}:refs/remotes/origin/${SEMVER}"]]]) - sh "./build.sh -t ${params.BUILD_TYPE} -p ${knowhere_build_dir} -j" - } - */ - dir ("cpp") { sh "git config --global user.email \"test@zilliz.com\"" sh "git config --global user.name \"test\"" - sh "./build.sh -t ${params.BUILD_TYPE} -j" + withCredentials([usernamePassword(credentialsId: "${params.JFROG_USER}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { + sh "export JFROG_ARTFACTORY_URL='http://192.168.1.201:80/artifactory' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -j" + } } } catch (exc) { updateGitlabCommitStatus name: 'Build Engine', state: 'failed' From 964ee0013dd7da483067cf9c2cb75e9b6f57b850 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Wed, 25 Sep 2019 17:42:21 +0800 Subject: [PATCH 5/5] add JFROG parameter Former-commit-id: 13493ae141c2ba1081b6e4c4eb5f9aa49a4b03ee --- ci/jenkinsfile/milvus_build.groovy | 2 +- ci/jenkinsfile/milvus_build_no_ut.groovy | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/jenkinsfile/milvus_build.groovy b/ci/jenkinsfile/milvus_build.groovy index 289b9a6fe8..44895ed5c9 100644 --- a/ci/jenkinsfile/milvus_build.groovy +++ b/ci/jenkinsfile/milvus_build.groovy @@ -9,7 +9,7 @@ container('milvus-build-env') { sh "git config --global user.email \"test@zilliz.com\"" sh "git config --global user.name \"test\"" withCredentials([usernamePassword(credentialsId: "${params.JFROG_USER}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh "export JFROG_ARTFACTORY_URL='http://192.168.1.201:80/artifactory' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -j -u -c" + sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -j -u -c" } } } catch (exc) { diff --git a/ci/jenkinsfile/milvus_build_no_ut.groovy b/ci/jenkinsfile/milvus_build_no_ut.groovy index 4caa9f0fa7..a807ec7ad4 100644 --- a/ci/jenkinsfile/milvus_build_no_ut.groovy +++ b/ci/jenkinsfile/milvus_build_no_ut.groovy @@ -9,7 +9,7 @@ container('milvus-build-env') { sh "git config --global user.email \"test@zilliz.com\"" sh "git config --global user.name \"test\"" withCredentials([usernamePassword(credentialsId: "${params.JFROG_USER}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh "export JFROG_ARTFACTORY_URL='http://192.168.1.201:80/artifactory' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -j" + sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -j" } } } catch (exc) {