diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index c1c09e01df..a92fdb71a6 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -40,6 +40,13 @@ jobs: docker login -u ${{ secrets.DOCKERHUB_USER }} \ -p ${{ secrets.DOCKERHUB_TOKEN }} docker-compose push ubuntu-core + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: ./ci/scripts/output_new.info + flags: unittests + yml: ./codecov.yaml centos: name: AMD64 CentOS ${{ matrix.centos }} @@ -70,3 +77,10 @@ jobs: docker login -u ${{ secrets.DOCKERHUB_USER }} \ -p ${{ secrets.DOCKERHUB_TOKEN }} docker-compose push centos-core + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v1 + with: + token: ${{ secrets.CODECOV_TOKEN }} + file: ./ci/scripts/output_new.info + flags: unittests + yml: ./codecov.yaml diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 12481eae12..aed986e012 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -48,7 +48,7 @@ pipeline { } axis { - name 'BINRARY_VERSION' + name 'BINARY_VERSION' values 'gpu', 'cpu' } } @@ -57,25 +57,25 @@ pipeline { stage("Milvus CI") { environment { PACKAGE_VERSION = VersionNumber([ - versionNumberString : '${SEMVER}-${BINRARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' + versionNumberString : '${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' ]); - DOCKER_VERSION = "${SEMVER}-${BINRARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}" + DOCKER_VERSION = "${SEMVER}-${BINARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}" } stages { stage('Run Build') { agent { kubernetes { - label "${OS_NAME}-${BINRARY_VERSION}-build" + label "${OS_NAME}-${BINARY_VERSION}-build-${env.BUILD_NUMBER}" defaultContainer 'jnlp' - yamlFile "ci/jenkins/pod/milvus-${BINRARY_VERSION}-version-${OS_NAME}-build-env-pod.yaml" + yamlFile "ci/jenkins/pod/milvus-${BINARY_VERSION}-version-${OS_NAME}-build-env-pod.yaml" } } stages { stage('Build') { steps { - container("milvus-${BINRARY_VERSION}-build-env") { + container("milvus-${BINARY_VERSION}-build-env") { script { load "${env.WORKSPACE}/ci/jenkins/step/build.groovy" } @@ -85,9 +85,9 @@ pipeline { stage('Code Coverage') { steps { - container("milvus-${BINRARY_VERSION}-build-env") { + container("milvus-${BINARY_VERSION}-build-env") { script { - if ("${BINRARY_VERSION}" == "gpu") { + if ("${BINARY_VERSION}" == "gpu") { load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" } else { echo "Skip Code Coverage" @@ -99,7 +99,7 @@ pipeline { stage('Upload Package') { steps { - container("milvus-${BINRARY_VERSION}-build-env") { + container("milvus-${BINARY_VERSION}-build-env") { script { load "${env.WORKSPACE}/ci/jenkins/step/package.groovy" } @@ -112,7 +112,7 @@ pipeline { stage('Publish docker images') { agent { kubernetes { - label "${OS_NAME}-${BINRARY_VERSION}-publish" + label "${OS_NAME}-${BINARY_VERSION}-publish-${env.BUILD_NUMBER}" defaultContainer 'jnlp' yamlFile 'ci/jenkins/pod/docker-pod.yaml' } @@ -135,12 +135,12 @@ pipeline { environment { FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-") FORMAT_OS_NAME = "${OS_NAME}".replaceAll("\\.", "-") - HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${FORMAT_OS_NAME}-${BINRARY_VERSION}".toLowerCase() + HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${FORMAT_OS_NAME}-${BINARY_VERSION}".toLowerCase() } agent { kubernetes { - label "${OS_NAME}-${BINRARY_VERSION}-dev-test" + label "${OS_NAME}-${BINARY_VERSION}-dev-test-${env.BUILD_NUMBER}" defaultContainer 'jnlp' yamlFile 'ci/jenkins/pod/testEnvironment.yaml' } @@ -171,19 +171,10 @@ pipeline { } } } - - stage('Cleanup Dev') { - steps { - container('milvus-test-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" - } - } - } - } } + post { - unsuccessful { + cleanup { container('milvus-test-env') { script { load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" diff --git a/ci/jenkins/step/build.groovy b/ci/jenkins/step/build.groovy index 6bf5052400..2a12929575 100644 --- a/ci/jenkins/step/build.groovy +++ b/ci/jenkins/step/build.groovy @@ -3,8 +3,8 @@ timeout(time: 60, unit: 'MINUTES') { withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { def checkResult = sh(script: "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache", returnStatus: true) - if ("${BINRARY_VERSION}" == "gpu") { - sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -g -u -c\"" + if ("${BINARY_VERSION}" == "gpu") { + sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -g -x -u -c\"" } else { sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -u -c\"" } diff --git a/ci/jenkins/step/deploySingle2Dev.groovy b/ci/jenkins/step/deploySingle2Dev.groovy index e566c3da0c..e39dbcfb40 100644 --- a/ci/jenkins/step/deploySingle2Dev.groovy +++ b/ci/jenkins/step/deploySingle2Dev.groovy @@ -3,7 +3,7 @@ sh 'helm repo update' dir ('milvus-helm') { checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]]) dir ("milvus") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/sqlite_${BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/sqlite_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } diff --git a/ci/jenkins/step/publishImages.groovy b/ci/jenkins/step/publishImages.groovy index fe1c211abc..885fc5cf47 100644 --- a/ci/jenkins/step/publishImages.groovy +++ b/ci/jenkins/step/publishImages.groovy @@ -1,5 +1,5 @@ timeout(time: 15, unit: 'MINUTES') { - dir ("docker/deploy/${BINRARY_VERSION}/${OS_NAME}") { + dir ("docker/deploy/${BINARY_VERSION}/${OS_NAME}") { def binaryPackage = "${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz" withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { diff --git a/ci/jenkins/step/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy index 618c2457b3..0949041b01 100644 --- a/ci/jenkins/step/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -13,7 +13,7 @@ timeout(time: 90, unit: 'MINUTES') { } dir ("milvus-helm") { dir ("milvus") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } dir ("tests/milvus_python_test") { diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index 255f748996..279052eac1 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -14,7 +14,7 @@ timeout(time: 60, unit: 'MINUTES') { // } // dir ("milvus-helm") { // dir ("milvus") { - // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${BINARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." // } // } // dir ("tests/milvus_python_test") { diff --git a/ci/jenkins/yaml/update_server_config.yaml b/ci/jenkins/yaml/update_server_config.yaml deleted file mode 100644 index 464365b577..0000000000 --- a/ci/jenkins/yaml/update_server_config.yaml +++ /dev/null @@ -1,2 +0,0 @@ -db_config: - primary_path: /var/lib/milvus diff --git a/ci/scripts/check_ccache.sh b/ci/scripts/check_ccache.sh index cc0708505e..c9c03ef149 100755 --- a/ci/scripts/check_ccache.sh +++ b/ci/scripts/check_ccache.sh @@ -44,7 +44,7 @@ check_ccache() { BRANCH=$1 PACKAGE_FILE="ccache-${OS_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" echo "fetching ${BRANCH}/${PACKAGE_FILE}" - wget -q --method HEAD "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" + wget -q --spider "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" if [[ $? == 0 ]];then wget -q "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" && \ mkdir -p ${CCACHE_DIRECTORY} && \ diff --git a/core/cmake/ThirdPartyPackages.cmake b/core/cmake/ThirdPartyPackages.cmake index 28ab35746c..fbd0b5b1bb 100644 --- a/core/cmake/ThirdPartyPackages.cmake +++ b/core/cmake/ThirdPartyPackages.cmake @@ -334,7 +334,7 @@ if (DEFINED ENV{MILVUS_GRPC_URL}) set(GRPC_SOURCE_URL "$ENV{MILVUS_GRPC_URL}") else () set(GRPC_SOURCE_URL - "https://github.com/ZhifengZhang-CN/grpc-milvus/archive/${GRPC_VERSION}.zip") + "https://github.com/youny626/grpc-milvus/archive/${GRPC_VERSION}.zip") endif () set(GRPC_MD5 "0362ba219f59432c530070b5f5c3df73") diff --git a/core/src/index/CMakeLists.txt b/core/src/index/CMakeLists.txt index 6e58566dd0..5664694155 100644 --- a/core/src/index/CMakeLists.txt +++ b/core/src/index/CMakeLists.txt @@ -98,9 +98,9 @@ if (CMAKE_BUILD_TYPE STREQUAL "Release") set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3") endif () else () - set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O3 -g -fPIC -DELPP_THREAD_SAFE -fopenmp -mavx -mf16c -msse4 -mpopcnt") + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -O0 -g -fPIC -DELPP_THREAD_SAFE -fopenmp -mavx -mf16c -msse4 -mpopcnt") if (KNOWHERE_GPU_VERSION) - set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O3 -g") + set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS} -O0 -g") endif () endif ()