diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 40d9686415..2298dcd918 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -33,128 +33,267 @@ pipeline { } stages { - stage("Ubuntu 18.04") { + stage("Ubuntu 18.04 x86_64") { environment { OS_NAME = "ubuntu18.04" - PACKAGE_VERSION = VersionNumber([ - versionNumberString : '${SEMVER}-${LOWER_BUILD_TYPE}-ubuntu18.04-x86_64-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' - ]); - DOCKER_VERSION = "${SEMVER}-${OS_NAME}-${LOWER_BUILD_TYPE}" + CPU_ARCH = "amd64" } - stages { - stage("Run Build") { - agent { - kubernetes { - label 'build' - defaultContainer 'jnlp' - yamlFile 'ci/jenkins/pod/milvus-build-env-pod.yaml' - } + parallel { + stage ("CPU Version") { + environment { + BINRARY_VERSION = "cpu" + PACKAGE_VERSION = VersionNumber([ + versionNumberString : '${SEMVER}-cpu-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' + ]); + DOCKER_VERSION = "${SEMVER}-cpu-${OS_NAME}-${LOWER_BUILD_TYPE}" } stages { - stage('Build') { - steps { - container('milvus-build-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/build.groovy" - } + stage("Run Build") { + agent { + kubernetes { + label 'build' + defaultContainer 'jnlp' + yamlFile 'ci/jenkins/pod/milvus-cpu-version-build-env-pod.yaml' } } - } - stage('Code Coverage') { - steps { - container('milvus-build-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" + + stages { + stage('Build') { + steps { + container('milvus-build-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/build.groovy" + } + } } } - } - } - stage('Upload Package') { - steps { - container('milvus-build-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/package.groovy" + stage('Code Coverage') { + steps { + container('milvus-build-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" + } + } } } - } - } - } - } - - stage("Publish docker images") { - agent { - kubernetes { - label 'publish' - defaultContainer 'jnlp' - yamlFile 'ci/jenkins/pod/docker-pod.yaml' - } - } - - stages { - stage('Publish') { - steps { - container('publish-images'){ - script { - load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy" - } - } - } - } - } - } - - stage("Deploy to Development") { - agent { - kubernetes { - label 'dev-test' - defaultContainer 'jnlp' - yamlFile 'ci/jenkins/pod/testEnvironment.yaml' - } - } - - stages { - stage("Deploy to Dev") { - steps { - container('milvus-test-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/deploySingle2Dev.groovy" - } - } - } - } - - stage("Dev Test") { - steps { - container('milvus-test-env') { - script { - boolean isNightlyTest = isTimeTriggeredBuild() - if (isNightlyTest) { - load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy" - } else { - load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy" + stage('Upload Package') { + steps { + container('milvus-build-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/package.groovy" + } } } } } } - stage ("Cleanup Dev") { - steps { - container('milvus-test-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" + stage("Publish docker images") { + agent { + kubernetes { + label 'publish' + defaultContainer 'jnlp' + yamlFile 'ci/jenkins/pod/docker-pod.yaml' + } + } + + stages { + stage('Publish') { + steps { + container('publish-images'){ + script { + load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy" + } + } + } + } + } + } + + stage("Deploy to Development") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yamlFile 'ci/jenkins/pod/testEnvironment.yaml' + } + } + + stages { + stage("Deploy to Dev") { + steps { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/deploySingle2Dev.groovy" + } + } + } + } + + stage("Dev Test") { + steps { + container('milvus-test-env') { + script { + boolean isNightlyTest = isTimeTriggeredBuild() + if (isNightlyTest) { + load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy" + } else { + load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy" + } + } + } + } + } + + stage ("Cleanup Dev") { + steps { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" + } + } + } + } + } + post { + unsuccessful { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" + } } } } } } - post { - unsuccessful { - container('milvus-test-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" + } + + stage ("GPU Version") { + environment { + BINRARY_VERSION = "gpu" + PACKAGE_VERSION = VersionNumber([ + versionNumberString : '${SEMVER}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-gpu-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' + ]); + DOCKER_VERSION = "${SEMVER}-${OS_NAME}-${LOWER_BUILD_TYPE}-gpu" + } + + stages { + stage("Run Build") { + agent { + kubernetes { + label 'build' + defaultContainer 'jnlp' + yamlFile 'ci/jenkins/pod/milvus-gpu-version-build-env-pod.yaml' + } + } + + stages { + stage('Build') { + steps { + container('milvus-build-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/build.groovy" + } + } + } + } + stage('Code Coverage') { + steps { + container('milvus-build-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" + } + } + } + } + stage('Upload Package') { + steps { + container('milvus-build-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/package.groovy" + } + } + } + } + } + } + + stage("Publish docker images") { + agent { + kubernetes { + label 'publish' + defaultContainer 'jnlp' + yamlFile 'ci/jenkins/pod/docker-pod.yaml' + } + } + + stages { + stage('Publish') { + steps { + container('publish-images'){ + script { + load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy" + } + } + } + } + } + } + + stage("Deploy to Development") { + agent { + kubernetes { + label 'dev-test' + defaultContainer 'jnlp' + yamlFile 'ci/jenkins/pod/testEnvironment.yaml' + } + } + + stages { + stage("Deploy to Dev") { + steps { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/deploySingle2Dev.groovy" + } + } + } + } + + stage("Dev Test") { + steps { + container('milvus-test-env') { + script { + boolean isNightlyTest = isTimeTriggeredBuild() + if (isNightlyTest) { + load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy" + } else { + load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy" + } + } + } + } + } + + stage ("Cleanup Dev") { + steps { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" + } + } + } + } + } + post { + unsuccessful { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" + } + } } } } diff --git a/ci/jenkins/pod/milvus-cpu-version-build-env-pod.yaml b/ci/jenkins/pod/milvus-cpu-version-build-env-pod.yaml new file mode 100644 index 0000000000..561bfe8140 --- /dev/null +++ b/ci/jenkins/pod/milvus-cpu-version-build-env-pod.yaml @@ -0,0 +1,34 @@ +apiVersion: v1 +kind: Pod +metadata: + name: milvus-cpu-build-env + labels: + app: milvus + componet: cpu-build-env +spec: + containers: + - name: milvus-build-env + image: registry.zilliz.com/milvus/milvus-cpu-build-env:v0.6.0-ubuntu18.04 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + command: + - cat + tty: true + resources: + limits: + memory: "32Gi" + cpu: "8.0" + requests: + memory: "16Gi" + cpu: "4.0" + - name: milvus-mysql + image: mysql:5.6 + env: + - name: MYSQL_ROOT_PASSWORD + value: 123456 + ports: + - containerPort: 3306 + name: mysql diff --git a/ci/jenkins/pod/milvus-build-env-pod.yaml b/ci/jenkins/pod/milvus-gpu-version-build-env-pod.yaml similarity index 80% rename from ci/jenkins/pod/milvus-build-env-pod.yaml rename to ci/jenkins/pod/milvus-gpu-version-build-env-pod.yaml index da938d8ba2..422dd72ab2 100644 --- a/ci/jenkins/pod/milvus-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-gpu-version-build-env-pod.yaml @@ -1,14 +1,14 @@ apiVersion: v1 kind: Pod metadata: - name: milvus-build-env + name: milvus-gpu-build-env labels: app: milvus - componet: build-env + componet: gpu-build-env spec: containers: - name: milvus-build-env - image: registry.zilliz.com/milvus/milvus-build-env:v0.5.1-ubuntu18.04 + image: registry.zilliz.com/milvus/milvus-gpu-build-env:v0.6.0-ubuntu18.04 env: - name: POD_IP valueFrom: diff --git a/ci/jenkins/step/build.groovy b/ci/jenkins/step/build.groovy index bae4259a6f..6c1da64a82 100644 --- a/ci/jenkins/step/build.groovy +++ b/ci/jenkins/step/build.groovy @@ -1,8 +1,11 @@ timeout(time: 60, unit: 'MINUTES') { dir ("ci/scripts") { withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { - sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -g -j -u -c" + if ("${env.BINRARY_VERSION}" == "gpu") { + sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -g -j -u -c" + } else { + sh "export JFROG_ARTFACTORY_URL='${params.JFROG_ARTFACTORY_URL}' && export JFROG_USER_NAME='${USERNAME}' && export JFROG_PASSWORD='${PASSWORD}' && ./build.sh -t ${params.BUILD_TYPE} -o /opt/milvus -l -m -j -u -c" + } } } } - diff --git a/ci/jenkins/step/cleanupSingleDev.groovy b/ci/jenkins/step/cleanupSingleDev.groovy index 3b8c1833b5..30325e0c91 100644 --- a/ci/jenkins/step/cleanupSingleDev.groovy +++ b/ci/jenkins/step/cleanupSingleDev.groovy @@ -1,12 +1,12 @@ try { - def helmResult = sh script: "helm status ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu", returnStatus: true + def helmResult = sh script: "helm status ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}", returnStatus: true if (!helmResult) { - sh "helm del --purge ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu" + sh "helm del --purge ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}" } } catch (exc) { - def helmResult = sh script: "helm status ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu", returnStatus: true + def helmResult = sh script: "helm status ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}", returnStatus: true if (!helmResult) { - sh "helm del --purge ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu" + sh "helm del --purge ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}" } throw exc } diff --git a/ci/jenkins/step/deploySingle2Dev.groovy b/ci/jenkins/step/deploySingle2Dev.groovy index f4964df5e2..1a81385acb 100644 --- a/ci/jenkins/step/deploySingle2Dev.groovy +++ b/ci/jenkins/step/deploySingle2Dev.groovy @@ -1,9 +1,13 @@ sh 'helm init --client-only --skip-refresh --stable-repo-url https://kubernetes.oss-cn-hangzhou.aliyuncs.com/charts' sh 'helm repo update' dir ('milvus-helm') { - checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]]) - dir ("milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/db_backend/sqlite_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]]) + dir ("milvus") { + if ("${env.BINRARY_VERSION}" == "gpu") { + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f gpu_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + } else { + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/filebeat/values.yaml --namespace milvus ." + } } } diff --git a/ci/jenkins/step/publishImages.groovy b/ci/jenkins/step/publishImages.groovy index 62df0c73bf..72e9924c62 100644 --- a/ci/jenkins/step/publishImages.groovy +++ b/ci/jenkins/step/publishImages.groovy @@ -1,6 +1,6 @@ container('publish-images') { timeout(time: 15, unit: 'MINUTES') { - dir ("docker/deploy/${OS_NAME}") { + dir ("docker/deploy/${env.BINRARY_VERSION}/${env.OS_NAME}") { def binaryPackage = "${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz" withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { diff --git a/ci/jenkins/step/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy index 9aeab2eb4e..7e837f97df 100644 --- a/ci/jenkins/step/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -1,22 +1,26 @@ timeout(time: 90, unit: 'MINUTES') { dir ("tests/milvus_python_test") { sh 'python3 -m pip install -r requirements.txt' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-milvus-engine.milvus.svc.cluster.local" } // mysql database backend test - load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" + // load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" - if (!fileExists('milvus-helm')) { - dir ("milvus-helm") { - checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]]) - } - } - dir ("milvus-helm") { - dir ("milvus-gpu") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." - } - } - dir ("tests/milvus_python_test") { - sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local" - } + // if (!fileExists('milvus-helm')) { + // dir ("milvus-helm") { + // checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]]) + // } + // } + // dir ("milvus-helm") { + // dir ("milvus") { + // if ("${env.BINRARY_VERSION}" == "gpu") { + // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f gpu_values.yaml -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + // } else { + // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + // } + // } + // } + // dir ("tests/milvus_python_test") { + // sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-milvus-engine.milvus.svc.cluster.local" + // } } diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index 86e6f126d9..ca681ccbdb 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -1,24 +1,27 @@ timeout(time: 60, unit: 'MINUTES') { dir ("tests/milvus_python_test") { sh 'python3 -m pip install -r requirements.txt' - sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local" + sh "pytest . --alluredir=\"test_out/dev/single/sqlite\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-milvus-engine.milvus.svc.cluster.local" } + // mysql database backend test // load "${env.WORKSPACE}/ci/jenkins/jenkinsfile/cleanupSingleDev.groovy" - // Remove mysql-version tests: 10-28 - // if (!fileExists('milvus-helm')) { // dir ("milvus-helm") { - // checkout([$class: 'GitSCM', branches: [[name: "0.5.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]]) + // checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.5.0:refs/remotes/origin/0.5.0"]]]) // } // } // dir ("milvus-helm") { - // dir ("milvus-gpu") { - // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + // dir ("milvus") { + // if ("${env.BINRARY_VERSION}" == "gpu") { + // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f gpu_values.yaml -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + // } else { + // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION} -f ci/db_backend/mysql_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + // } // } // } // dir ("tests/milvus_python_test") { - // sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-gpu-milvus-gpu-engine.milvus.svc.cluster.local" + // sh "pytest . --alluredir=\"test_out/dev/single/mysql\" --level=1 --ip ${env.PIPELINE_NAME}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}-milvus-engine.milvus.svc.cluster.local" // } } diff --git a/docker/build_env/cpu/ubuntu16.04/Dockerfile b/docker/build_env/cpu/ubuntu16.04/Dockerfile new file mode 100644 index 0000000000..6979ae9d40 --- /dev/null +++ b/docker/build_env/cpu/ubuntu16.04/Dockerfile @@ -0,0 +1,26 @@ +FROM ubuntu:16.04 + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends wget ca-certificates gnupg2 apt-transport-https && \ + wget -P /tmp https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ + apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ + sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \ + wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ + apt-get update && apt-get install -y --no-install-recommends \ + g++ git gfortran lsb-core \ + libboost-serialization-dev libboost-filesystem-dev libboost-system-dev libboost-regex-dev \ + curl libtool automake libssl-dev pkg-config libcurl4-openssl-dev python3-pip \ + clang-format-6.0 clang-tidy-6.0 \ + lcov mysql-client libmysqlclient-dev intel-mkl-gnu-2019.5-281 intel-mkl-core-2019.5-281 && \ + apt-get remove --purge -y && \ + rm -rf /var/lib/apt/lists/* + +RUN ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so \ + /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so + +RUN sh -c 'echo export LD_LIBRARY_PATH=/opt/intel/compilers_and_libraries_2019.5.281/linux/mkl/lib/intel64:\$LD_LIBRARY_PATH > /etc/profile.d/mkl.sh' + +COPY docker-entrypoint.sh /app/docker-entrypoint.sh +ENTRYPOINT [ "/app/docker-entrypoint.sh" ] +CMD [ "start" ] diff --git a/docker/build_env/ubuntu16.04/docker-entrypoint.sh b/docker/build_env/cpu/ubuntu16.04/docker-entrypoint.sh similarity index 100% rename from docker/build_env/ubuntu16.04/docker-entrypoint.sh rename to docker/build_env/cpu/ubuntu16.04/docker-entrypoint.sh diff --git a/docker/build_env/cpu/ubuntu18.04/Dockerfile b/docker/build_env/cpu/ubuntu18.04/Dockerfile new file mode 100644 index 0000000000..6cd2bfc1c7 --- /dev/null +++ b/docker/build_env/cpu/ubuntu18.04/Dockerfile @@ -0,0 +1,26 @@ +FROM ubuntu:18.04 + +ENV DEBIAN_FRONTEND noninteractive + +RUN apt-get update && apt-get install -y --no-install-recommends wget ca-certificates gnupg2 && \ + wget -P /tmp https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ + apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB && \ + sh -c 'echo deb https://apt.repos.intel.com/mkl all main > /etc/apt/sources.list.d/intel-mkl.list' && \ + wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ + apt-get update && apt-get install -y --no-install-recommends \ + g++ git gfortran lsb-core \ + libboost-serialization-dev libboost-filesystem-dev libboost-system-dev libboost-regex-dev \ + curl libtool automake libssl-dev pkg-config libcurl4-openssl-dev python3-pip \ + clang-format-6.0 clang-tidy-6.0 \ + lcov mysql-client libmysqlclient-dev intel-mkl-gnu-2019.5-281 intel-mkl-core-2019.5-281 && \ + apt-get remove --purge -y && \ + rm -rf /var/lib/apt/lists/* + +RUN ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so \ + /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so + +RUN sh -c 'echo export LD_LIBRARY_PATH=/opt/intel/compilers_and_libraries_2019.5.281/linux/mkl/lib/intel64:\$LD_LIBRARY_PATH > /etc/profile.d/mkl.sh' + +COPY docker-entrypoint.sh /app/docker-entrypoint.sh +ENTRYPOINT [ "/app/docker-entrypoint.sh" ] +CMD [ "start" ] diff --git a/docker/build_env/ubuntu18.04/docker-entrypoint.sh b/docker/build_env/cpu/ubuntu18.04/docker-entrypoint.sh similarity index 100% rename from docker/build_env/ubuntu18.04/docker-entrypoint.sh rename to docker/build_env/cpu/ubuntu18.04/docker-entrypoint.sh diff --git a/docker/build_env/ubuntu16.04/Dockerfile b/docker/build_env/gpu/ubuntu16.04/Dockerfile similarity index 100% rename from docker/build_env/ubuntu16.04/Dockerfile rename to docker/build_env/gpu/ubuntu16.04/Dockerfile diff --git a/docker/build_env/gpu/ubuntu16.04/docker-entrypoint.sh b/docker/build_env/gpu/ubuntu16.04/docker-entrypoint.sh new file mode 100755 index 0000000000..1e85e7e9e1 --- /dev/null +++ b/docker/build_env/gpu/ubuntu16.04/docker-entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +if [ "$1" = 'start' ]; then + tail -f /dev/null +fi + +exec "$@" + diff --git a/docker/build_env/ubuntu18.04/Dockerfile b/docker/build_env/gpu/ubuntu18.04/Dockerfile similarity index 100% rename from docker/build_env/ubuntu18.04/Dockerfile rename to docker/build_env/gpu/ubuntu18.04/Dockerfile diff --git a/docker/build_env/gpu/ubuntu18.04/docker-entrypoint.sh b/docker/build_env/gpu/ubuntu18.04/docker-entrypoint.sh new file mode 100755 index 0000000000..1e85e7e9e1 --- /dev/null +++ b/docker/build_env/gpu/ubuntu18.04/docker-entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +if [ "$1" = 'start' ]; then + tail -f /dev/null +fi + +exec "$@" + diff --git a/docker/deploy/cpu/ubuntu16.04/Dockerfile b/docker/deploy/cpu/ubuntu16.04/Dockerfile new file mode 100644 index 0000000000..2a8317733b --- /dev/null +++ b/docker/deploy/cpu/ubuntu16.04/Dockerfile @@ -0,0 +1,18 @@ +FROM ubuntu:16.04 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + gfortran libsqlite3-dev libmysqlclient-dev libcurl4-openssl-dev python3 && \ + apt-get remove --purge -y && \ + rm -rf /var/lib/apt/lists/* + +RUN ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so + +COPY ./docker-entrypoint.sh /opt +COPY ./milvus /opt/milvus +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/milvus/lib" + +ENTRYPOINT [ "/opt/docker-entrypoint.sh" ] + +CMD [ "start" ] + +EXPOSE 19530 diff --git a/docker/deploy/ubuntu16.04/docker-entrypoint.sh b/docker/deploy/cpu/ubuntu16.04/docker-entrypoint.sh similarity index 99% rename from docker/deploy/ubuntu16.04/docker-entrypoint.sh rename to docker/deploy/cpu/ubuntu16.04/docker-entrypoint.sh index 446c174d74..12937df395 100755 --- a/docker/deploy/ubuntu16.04/docker-entrypoint.sh +++ b/docker/deploy/cpu/ubuntu16.04/docker-entrypoint.sh @@ -7,4 +7,3 @@ if [ "$1" == 'start' ]; then fi exec "$@" - diff --git a/docker/deploy/cpu/ubuntu18.04/Dockerfile b/docker/deploy/cpu/ubuntu18.04/Dockerfile new file mode 100644 index 0000000000..4cf9ceebad --- /dev/null +++ b/docker/deploy/cpu/ubuntu18.04/Dockerfile @@ -0,0 +1,19 @@ +FROM ubuntu:18.04 + +RUN apt-get update && apt-get install -y --no-install-recommends \ + gfortran libsqlite3-dev libmysqlclient-dev libcurl4-openssl-dev python3 && \ + apt-get remove --purge -y && \ + rm -rf /var/lib/apt/lists/* + +RUN ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so + +COPY ./docker-entrypoint.sh /opt +COPY ./milvus /opt/milvus +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/opt/milvus/lib" + +ENTRYPOINT [ "/opt/docker-entrypoint.sh" ] + +CMD [ "start" ] + +EXPOSE 19530 + diff --git a/docker/deploy/ubuntu18.04/docker-entrypoint.sh b/docker/deploy/cpu/ubuntu18.04/docker-entrypoint.sh similarity index 99% rename from docker/deploy/ubuntu18.04/docker-entrypoint.sh rename to docker/deploy/cpu/ubuntu18.04/docker-entrypoint.sh index 446c174d74..12937df395 100755 --- a/docker/deploy/ubuntu18.04/docker-entrypoint.sh +++ b/docker/deploy/cpu/ubuntu18.04/docker-entrypoint.sh @@ -7,4 +7,3 @@ if [ "$1" == 'start' ]; then fi exec "$@" - diff --git a/docker/deploy/ubuntu16.04/Dockerfile b/docker/deploy/gpu/ubuntu16.04/Dockerfile similarity index 100% rename from docker/deploy/ubuntu16.04/Dockerfile rename to docker/deploy/gpu/ubuntu16.04/Dockerfile diff --git a/docker/deploy/gpu/ubuntu16.04/docker-entrypoint.sh b/docker/deploy/gpu/ubuntu16.04/docker-entrypoint.sh new file mode 100755 index 0000000000..12937df395 --- /dev/null +++ b/docker/deploy/gpu/ubuntu16.04/docker-entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e + +if [ "$1" == 'start' ]; then + cd /opt/milvus/scripts && ./start_server.sh +fi + +exec "$@" diff --git a/docker/deploy/ubuntu18.04/Dockerfile b/docker/deploy/gpu/ubuntu18.04/Dockerfile similarity index 100% rename from docker/deploy/ubuntu18.04/Dockerfile rename to docker/deploy/gpu/ubuntu18.04/Dockerfile diff --git a/docker/deploy/gpu/ubuntu18.04/docker-entrypoint.sh b/docker/deploy/gpu/ubuntu18.04/docker-entrypoint.sh new file mode 100755 index 0000000000..12937df395 --- /dev/null +++ b/docker/deploy/gpu/ubuntu18.04/docker-entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e + +if [ "$1" == 'start' ]; then + cd /opt/milvus/scripts && ./start_server.sh +fi + +exec "$@"