From a6e774aed6994630816dd255d0d896cc7ab0fba4 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Fri, 13 Dec 2019 21:21:33 +0800 Subject: [PATCH 01/34] enable build milvus on centos7 --- .../index/cmake/ThirdPartyPackagesCore.cmake | 7 +++--- docker/build_env/gpu/centos7/Dockerfile | 22 +++++++++++++++++++ .../gpu/centos7/docker-entrypoint.sh | 10 +++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 docker/build_env/gpu/centos7/Dockerfile create mode 100755 docker/build_env/gpu/centos7/docker-entrypoint.sh diff --git a/core/src/index/cmake/ThirdPartyPackagesCore.cmake b/core/src/index/cmake/ThirdPartyPackagesCore.cmake index 9c9187d2cc..9ae0391f54 100644 --- a/core/src/index/cmake/ThirdPartyPackagesCore.cmake +++ b/core/src/index/cmake/ThirdPartyPackagesCore.cmake @@ -270,10 +270,10 @@ set(ARROW_PREFIX "${INDEX_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep/cpp") macro(build_arrow) message(STATUS "Building Apache ARROW-${ARROW_VERSION} from source") set(ARROW_STATIC_LIB_NAME arrow) - set(ARROW_STATIC_LIB - "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) set(ARROW_LIB_DIR "${ARROW_PREFIX}/lib") + set(ARROW_STATIC_LIB + "${ARROW_LIB_DIR}/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}" + ) set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include") set(ARROW_CMAKE_ARGS @@ -282,6 +282,7 @@ macro(build_arrow) -DARROW_BUILD_SHARED=OFF -DARROW_USE_GLOG=OFF -DCMAKE_INSTALL_PREFIX=${ARROW_PREFIX} + -DCMAKE_INSTALL_LIBDIR=${ARROW_LIB_DIR} -DARROW_CUDA=OFF -DARROW_FLIGHT=OFF -DARROW_GANDIVA=OFF diff --git a/docker/build_env/gpu/centos7/Dockerfile b/docker/build_env/gpu/centos7/Dockerfile new file mode 100644 index 0000000000..0f8ef6c85c --- /dev/null +++ b/docker/build_env/gpu/centos7/Dockerfile @@ -0,0 +1,22 @@ +FROM nvidia/cuda:10.1-devel-centos7 + +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility + +RUN yum install -y epel-release centos-release-scl-rh && yum install -y wget curl && \ + wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ + yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static devtoolset-7-gcc llvm-toolset-7.0-clang lcov \ + && \ + rm -rf /var/cache/yum/* + +RUN scl enable devtoolset-7 bash + +ENV CLANG_TOOLS_PATH="/opt/rh/llvm-toolset-7.0/root/usr/bin" +ENV PATH="/opt/rh/llvm-toolset-7.0/root/usr/bin:$PATH" +ENV LD_LIBRARY_PATH="/opt/rh/llvm-toolset-7.0/root/usr/lib64:$LD_LIBRARY_PATH" + +COPY docker-entrypoint.sh /app/docker-entrypoint.sh + +WORKDIR /root + +ENTRYPOINT [ "/app/docker-entrypoint.sh" ] +CMD [ "start" ] diff --git a/docker/build_env/gpu/centos7/docker-entrypoint.sh b/docker/build_env/gpu/centos7/docker-entrypoint.sh new file mode 100755 index 0000000000..1e85e7e9e1 --- /dev/null +++ b/docker/build_env/gpu/centos7/docker-entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +if [ "$1" = 'start' ]; then + tail -f /dev/null +fi + +exec "$@" + From cfa9791d9c9f551b221f1217cb2a3e93b38e49c4 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Fri, 13 Dec 2019 21:55:13 +0800 Subject: [PATCH 02/34] Update build enviroment Centos7 dockerfile --- docker/build_env/gpu/centos7/Dockerfile | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docker/build_env/gpu/centos7/Dockerfile b/docker/build_env/gpu/centos7/Dockerfile index 0f8ef6c85c..28bffffdcd 100644 --- a/docker/build_env/gpu/centos7/Dockerfile +++ b/docker/build_env/gpu/centos7/Dockerfile @@ -4,15 +4,14 @@ ENV NVIDIA_DRIVER_CAPABILITIES compute,utility RUN yum install -y epel-release centos-release-scl-rh && yum install -y wget curl && \ wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ - yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static devtoolset-7-gcc llvm-toolset-7.0-clang lcov \ + yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static devtoolset-6-gcc llvm-toolset-7.0-clang lcov \ && \ rm -rf /var/cache/yum/* -RUN scl enable devtoolset-7 bash +RUN echo "source scl_source enable devtoolset-6" >> /etc/profile.d/devtoolset-6.sh +RUN echo "source scl_source enable llvm-toolset-7" >> /etc/profile.d/llvm-toolset-7.sh ENV CLANG_TOOLS_PATH="/opt/rh/llvm-toolset-7.0/root/usr/bin" -ENV PATH="/opt/rh/llvm-toolset-7.0/root/usr/bin:$PATH" -ENV LD_LIBRARY_PATH="/opt/rh/llvm-toolset-7.0/root/usr/lib64:$LD_LIBRARY_PATH" COPY docker-entrypoint.sh /app/docker-entrypoint.sh From 14141e3474a26c28bb7ad5c4443e1aef7933a12e Mon Sep 17 00:00:00 2001 From: quicksilver Date: Fri, 13 Dec 2019 21:58:10 +0800 Subject: [PATCH 03/34] Update Dockerfile --- docker/build_env/gpu/centos7/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/build_env/gpu/centos7/Dockerfile b/docker/build_env/gpu/centos7/Dockerfile index 28bffffdcd..9790a4b752 100644 --- a/docker/build_env/gpu/centos7/Dockerfile +++ b/docker/build_env/gpu/centos7/Dockerfile @@ -4,11 +4,11 @@ ENV NVIDIA_DRIVER_CAPABILITIES compute,utility RUN yum install -y epel-release centos-release-scl-rh && yum install -y wget curl && \ wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ - yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static devtoolset-6-gcc llvm-toolset-7.0-clang lcov \ + yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static devtoolset-7-gcc llvm-toolset-7.0-clang lcov \ && \ rm -rf /var/cache/yum/* -RUN echo "source scl_source enable devtoolset-6" >> /etc/profile.d/devtoolset-6.sh +RUN echo "source scl_source enable devtoolset-7" >> /etc/profile.d/devtoolset-7.sh RUN echo "source scl_source enable llvm-toolset-7" >> /etc/profile.d/llvm-toolset-7.sh ENV CLANG_TOOLS_PATH="/opt/rh/llvm-toolset-7.0/root/usr/bin" From f4d639767dff221876ba832c5deda7aa73a4a9f0 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Fri, 13 Dec 2019 22:00:19 +0800 Subject: [PATCH 04/34] Update Dockerfile --- docker/build_env/gpu/centos7/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/build_env/gpu/centos7/Dockerfile b/docker/build_env/gpu/centos7/Dockerfile index 9790a4b752..9b5148cdac 100644 --- a/docker/build_env/gpu/centos7/Dockerfile +++ b/docker/build_env/gpu/centos7/Dockerfile @@ -9,7 +9,7 @@ RUN yum install -y epel-release centos-release-scl-rh && yum install -y wget cur rm -rf /var/cache/yum/* RUN echo "source scl_source enable devtoolset-7" >> /etc/profile.d/devtoolset-7.sh -RUN echo "source scl_source enable llvm-toolset-7" >> /etc/profile.d/llvm-toolset-7.sh +RUN echo "source scl_source enable llvm-toolset-7.0" >> /etc/profile.d/llvm-toolset-7.sh ENV CLANG_TOOLS_PATH="/opt/rh/llvm-toolset-7.0/root/usr/bin" From dcd358222f46bf5e2c1be4f2781ad72aff7ce957 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Fri, 13 Dec 2019 22:03:16 +0800 Subject: [PATCH 05/34] Update Dockerfile --- docker/build_env/gpu/centos7/Dockerfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker/build_env/gpu/centos7/Dockerfile b/docker/build_env/gpu/centos7/Dockerfile index 9b5148cdac..16f6225fb4 100644 --- a/docker/build_env/gpu/centos7/Dockerfile +++ b/docker/build_env/gpu/centos7/Dockerfile @@ -4,7 +4,8 @@ ENV NVIDIA_DRIVER_CAPABILITIES compute,utility RUN yum install -y epel-release centos-release-scl-rh && yum install -y wget curl && \ wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ - yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static devtoolset-7-gcc llvm-toolset-7.0-clang lcov \ + yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static mysql-devel \ + devtoolset-7-gcc llvm-toolset-7.0-clang lcov \ && \ rm -rf /var/cache/yum/* From baee59901e3c9754fd0e51c1300fe73d14fdaf7b Mon Sep 17 00:00:00 2001 From: quicksilver Date: Fri, 13 Dec 2019 22:10:15 +0800 Subject: [PATCH 06/34] Update Dockerfile --- docker/build_env/gpu/centos7/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/build_env/gpu/centos7/Dockerfile b/docker/build_env/gpu/centos7/Dockerfile index 16f6225fb4..fc006da02d 100644 --- a/docker/build_env/gpu/centos7/Dockerfile +++ b/docker/build_env/gpu/centos7/Dockerfile @@ -5,7 +5,7 @@ ENV NVIDIA_DRIVER_CAPABILITIES compute,utility RUN yum install -y epel-release centos-release-scl-rh && yum install -y wget curl && \ wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static mysql-devel \ - devtoolset-7-gcc llvm-toolset-7.0-clang lcov \ + devtoolset-7-gcc devtoolset-7-gcc-c++ llvm-toolset-7.0-clang lcov \ && \ rm -rf /var/cache/yum/* From 10c5093781009e74204c7d1c8d194dce41ddaf6a Mon Sep 17 00:00:00 2001 From: quicksilver Date: Fri, 13 Dec 2019 22:27:24 +0800 Subject: [PATCH 07/34] Update Dockerfile --- docker/build_env/gpu/centos7/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/build_env/gpu/centos7/Dockerfile b/docker/build_env/gpu/centos7/Dockerfile index fc006da02d..4e5119bab9 100644 --- a/docker/build_env/gpu/centos7/Dockerfile +++ b/docker/build_env/gpu/centos7/Dockerfile @@ -5,11 +5,11 @@ ENV NVIDIA_DRIVER_CAPABILITIES compute,utility RUN yum install -y epel-release centos-release-scl-rh && yum install -y wget curl && \ wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static mysql-devel \ - devtoolset-7-gcc devtoolset-7-gcc-c++ llvm-toolset-7.0-clang lcov \ + devtoolset-6-gcc devtoolset-6-gcc-c++ llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra lcov \ && \ rm -rf /var/cache/yum/* -RUN echo "source scl_source enable devtoolset-7" >> /etc/profile.d/devtoolset-7.sh +RUN echo "source scl_source enable devtoolset-6" >> /etc/profile.d/devtoolset-6.sh RUN echo "source scl_source enable llvm-toolset-7.0" >> /etc/profile.d/llvm-toolset-7.sh ENV CLANG_TOOLS_PATH="/opt/rh/llvm-toolset-7.0/root/usr/bin" From cde8b3d0088e822920cc9956a3ff2e1073445956 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Fri, 13 Dec 2019 23:12:51 +0800 Subject: [PATCH 08/34] Update Dockerfile --- docker/build_env/gpu/centos7/Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docker/build_env/gpu/centos7/Dockerfile b/docker/build_env/gpu/centos7/Dockerfile index 4e5119bab9..adeac7a65a 100644 --- a/docker/build_env/gpu/centos7/Dockerfile +++ b/docker/build_env/gpu/centos7/Dockerfile @@ -5,11 +5,11 @@ ENV NVIDIA_DRIVER_CAPABILITIES compute,utility RUN yum install -y epel-release centos-release-scl-rh && yum install -y wget curl && \ wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static mysql-devel \ - devtoolset-6-gcc devtoolset-6-gcc-c++ llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra lcov \ + devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-gcc-gfortran llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra lcov \ && \ rm -rf /var/cache/yum/* -RUN echo "source scl_source enable devtoolset-6" >> /etc/profile.d/devtoolset-6.sh +RUN echo "source scl_source enable devtoolset-7" >> /etc/profile.d/devtoolset-7.sh RUN echo "source scl_source enable llvm-toolset-7.0" >> /etc/profile.d/llvm-toolset-7.sh ENV CLANG_TOOLS_PATH="/opt/rh/llvm-toolset-7.0/root/usr/bin" From 0444062e325bae6055d65ef4a76fdafc50e824d0 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Fri, 13 Dec 2019 23:50:46 +0800 Subject: [PATCH 09/34] Update Dockerfile --- docker/build_env/gpu/centos7/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/build_env/gpu/centos7/Dockerfile b/docker/build_env/gpu/centos7/Dockerfile index adeac7a65a..b65ce1df6e 100644 --- a/docker/build_env/gpu/centos7/Dockerfile +++ b/docker/build_env/gpu/centos7/Dockerfile @@ -2,7 +2,7 @@ FROM nvidia/cuda:10.1-devel-centos7 ENV NVIDIA_DRIVER_CAPABILITIES compute,utility -RUN yum install -y epel-release centos-release-scl-rh && yum install -y wget curl && \ +RUN yum install -y epel-release centos-release-scl-rh && yum install -y wget curl which && \ wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static mysql-devel \ devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-gcc-gfortran llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra lcov \ From 03af301e0c9f458764c6736fe1fffa8be36e6977 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sat, 14 Dec 2019 00:29:04 +0800 Subject: [PATCH 10/34] add centos7_build_deps.sh --- core/centos7_build_deps.sh | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100755 core/centos7_build_deps.sh diff --git a/core/centos7_build_deps.sh b/core/centos7_build_deps.sh new file mode 100755 index 0000000000..f415e359af --- /dev/null +++ b/core/centos7_build_deps.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +sudo yum install -y epel-release centos-release-scl-rh && sudo yum install -y wget curl which && \ +sudo wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | sudo tar --strip-components=1 -xz -C /usr/local && \ +sudo yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static mysql-devel \ +devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-gcc-gfortran llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra lcov + +echo "source scl_source enable devtoolset-7" | sudo tee -a /etc/profile.d/devtoolset-7.sh +echo "source scl_source enable llvm-toolset-7.0" | sudo tee -a /etc/profile.d/llvm-toolset-7.sh +echo "export CLANG_TOOLS_PATH=/opt/rh/llvm-toolset-7.0/root/usr/bin" | sudo tee -a /etc/profile.d/llvm-toolset-7.sh + From ca37da59a28b1348403d6c637f3ba037c347705c Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sat, 14 Dec 2019 11:19:11 +0800 Subject: [PATCH 11/34] add centos7 build cpu version enviroment --- core/unittest/server/test_config.cpp | 5 +++-- docker/build_env/cpu/centos7/Dockerfile | 20 +++++++++++++++++++ .../cpu/centos7/docker-entrypoint.sh | 10 ++++++++++ 3 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 docker/build_env/cpu/centos7/Dockerfile create mode 100755 docker/build_env/cpu/centos7/docker-entrypoint.sh diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 791876ee8b..8705555d12 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -17,6 +17,7 @@ #include #include +#include #include "config/YamlConfigMgr.h" #include "server/Config.h" @@ -67,9 +68,9 @@ TEST_F(ConfigTest, CONFIG_TEST) { server_config.SetValue("float_test", "2.5"); double dbl = server_config.GetDoubleValue("float_test"); - ASSERT_LE(abs(dbl - 2.5), std::numeric_limits::epsilon()); + ASSERT_LE(std:fabs(dbl - 2.5), std::numeric_limits::epsilon()); float flt = server_config.GetFloatValue("float_test"); - ASSERT_LE(abs(flt - 2.5), std::numeric_limits::epsilon()); + ASSERT_LE(std:fabs(flt - 2.5), std::numeric_limits::epsilon()); server_config.SetValue("bool_test", "true"); bool blt = server_config.GetBoolValue("bool_test"); diff --git a/docker/build_env/cpu/centos7/Dockerfile b/docker/build_env/cpu/centos7/Dockerfile new file mode 100644 index 0000000000..2e564b0323 --- /dev/null +++ b/docker/build_env/cpu/centos7/Dockerfile @@ -0,0 +1,20 @@ +FROM centos:centos7 + +RUN yum install -y epel-release centos-release-scl-rh && yum install -y wget curl which && \ + wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ + yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static mysql-devel \ + devtoolset-6-gcc devtoolset-6-gcc-c++ devtoolset-6-gcc-gfortran llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra lcov \ + && \ + rm -rf /var/cache/yum/* + +RUN echo "source scl_source enable devtoolset-6" >> /etc/profile.d/devtoolset-6.sh +RUN echo "source scl_source enable llvm-toolset-7.0" >> /etc/profile.d/llvm-toolset-7.sh + +ENV CLANG_TOOLS_PATH="/opt/rh/llvm-toolset-7.0/root/usr/bin" + +COPY docker-entrypoint.sh /app/docker-entrypoint.sh + +WORKDIR /root + +ENTRYPOINT [ "/app/docker-entrypoint.sh" ] +CMD [ "start" ] \ No newline at end of file diff --git a/docker/build_env/cpu/centos7/docker-entrypoint.sh b/docker/build_env/cpu/centos7/docker-entrypoint.sh new file mode 100755 index 0000000000..1e85e7e9e1 --- /dev/null +++ b/docker/build_env/cpu/centos7/docker-entrypoint.sh @@ -0,0 +1,10 @@ +#!/bin/bash + +set -e + +if [ "$1" = 'start' ]; then + tail -f /dev/null +fi + +exec "$@" + From ad299b7b40093353584b56f39061ce8899bc452e Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sat, 14 Dec 2019 11:39:52 +0800 Subject: [PATCH 12/34] add centos7 on github actions --- .github/workflows/core.yml | 30 ++++++++++++++++++++++++++++++ ci/docker/centos-7-core.dockerfile | 14 ++++++++++++++ docker-compose.yml | 18 ++++++++++++++++++ 3 files changed, 62 insertions(+) create mode 100644 ci/docker/centos-7-core.dockerfile diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index ab26f0b189..c1c09e01df 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -40,3 +40,33 @@ jobs: docker login -u ${{ secrets.DOCKERHUB_USER }} \ -p ${{ secrets.DOCKERHUB_TOKEN }} docker-compose push ubuntu-core + + centos: + name: AMD64 CentOS ${{ matrix.centos }} + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + centos: [7] + env: + CENTOS: ${{ matrix.centos }} + steps: + - name: Checkout Milvus + uses: actions/checkout@v1 + - name: Docker Pull + shell: bash + run: | + docker-compose pull --ignore-pull-failures db + docker-compose pull --ignore-pull-failures centos-core + - name: Docker Build + run: | + docker-compose run --use-aliases -d db + docker-compose run centos-core + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' + continue-on-error: true + shell: bash + run: | + docker login -u ${{ secrets.DOCKERHUB_USER }} \ + -p ${{ secrets.DOCKERHUB_TOKEN }} + docker-compose push centos-core diff --git a/ci/docker/centos-7-core.dockerfile b/ci/docker/centos-7-core.dockerfile new file mode 100644 index 0000000000..8b3afce43e --- /dev/null +++ b/ci/docker/centos-7-core.dockerfile @@ -0,0 +1,14 @@ +ARG arch=amd64 +FROM ${arch}/centos:7 + +RUN yum install -y epel-release centos-release-scl-rh && yum install -y wget curl which && \ + wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ + yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static mysql-devel \ + devtoolset-6-gcc devtoolset-6-gcc-c++ devtoolset-6-gcc-gfortran llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra lcov \ + && \ + rm -rf /var/cache/yum/* + +RUN echo "source scl_source enable devtoolset-6" >> /etc/profile.d/devtoolset-6.sh +RUN echo "source scl_source enable llvm-toolset-7.0" >> /etc/profile.d/llvm-toolset-7.sh + +ENV CLANG_TOOLS_PATH="/opt/rh/llvm-toolset-7.0/root/usr/bin" diff --git a/docker-compose.yml b/docker-compose.yml index 19483016b8..0b3c4173c4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -2,6 +2,7 @@ version: '3.5' volumes: amd64-ubuntu-18.04-cache: + amd64-centos-7-cache: x-ccache: &ccache CCACHE_COMPILERCHECK: content @@ -44,5 +45,22 @@ services: - milvus command: *cpp-command + centos-core: + image: ${REPO}:${ARCH}-centos-${CENTOS}-core + build: + context: . + dockerfile: ci/docker/centos-${CENTOS}-core.dockerfile + cache_from: + - ${REPO}:${ARCH}-centos-${CENTOS}-core + shm_size: 2G + environment: + <<: *ccache + volumes: &ubuntu-volumes + - .:/milvus:delegated + - ${ARCH}-centos-${CENTOS}-cache:/build:delegated + networks: + - milvus + command: *cpp-command + networks: milvus: From b8133a7316bb1d71e075cc2b94b92dbba22ab560 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sat, 14 Dec 2019 11:41:29 +0800 Subject: [PATCH 13/34] fix bug --- core/unittest/server/test_config.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/unittest/server/test_config.cpp b/core/unittest/server/test_config.cpp index 8705555d12..bf6091815a 100644 --- a/core/unittest/server/test_config.cpp +++ b/core/unittest/server/test_config.cpp @@ -68,9 +68,9 @@ TEST_F(ConfigTest, CONFIG_TEST) { server_config.SetValue("float_test", "2.5"); double dbl = server_config.GetDoubleValue("float_test"); - ASSERT_LE(std:fabs(dbl - 2.5), std::numeric_limits::epsilon()); + ASSERT_LE(std::fabs(dbl - 2.5), std::numeric_limits::epsilon()); float flt = server_config.GetFloatValue("float_test"); - ASSERT_LE(std:fabs(flt - 2.5), std::numeric_limits::epsilon()); + ASSERT_LE(std::fabs(flt - 2.5), std::numeric_limits::epsilon()); server_config.SetValue("bool_test", "true"); bool blt = server_config.GetBoolValue("bool_test"); From 3ef184340274afa4e242164b246bb9a04e3b728f Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sat, 14 Dec 2019 11:42:58 +0800 Subject: [PATCH 14/34] fix bug --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 0b3c4173c4..01adaf7292 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -55,7 +55,7 @@ services: shm_size: 2G environment: <<: *ccache - volumes: &ubuntu-volumes + volumes: ¢os-volumes - .:/milvus:delegated - ${ARCH}-centos-${CENTOS}-cache:/build:delegated networks: From 660b3f3bd1debc644254c8abebef70796ebeb0f4 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sat, 14 Dec 2019 11:45:51 +0800 Subject: [PATCH 15/34] fix bug --- .env | 1 + 1 file changed, 1 insertion(+) diff --git a/.env b/.env index bcb772c2de..3299802e25 100644 --- a/.env +++ b/.env @@ -2,4 +2,5 @@ REPO=milvusdb/milvus-dev MILVUS_INSTALL_PREFIX=/var/lib/milvus ARCH=amd64 UBUNTU=18.04 +CENTOS=7 CUDA=10.1 \ No newline at end of file From 529f2f9b4ad6797493baf6704a4fab3e9a801f01 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sat, 14 Dec 2019 11:49:00 +0800 Subject: [PATCH 16/34] update ci/docker/centos-7-core.dockerfile --- ci/docker/centos-7-core.dockerfile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ci/docker/centos-7-core.dockerfile b/ci/docker/centos-7-core.dockerfile index 8b3afce43e..b6da6a4989 100644 --- a/ci/docker/centos-7-core.dockerfile +++ b/ci/docker/centos-7-core.dockerfile @@ -1,6 +1,10 @@ ARG arch=amd64 FROM ${arch}/centos:7 +# pipefail is enabled for proper error detection in the `wget` +# step +SHELL ["/bin/bash", "-o", "pipefail", "-c"] + RUN yum install -y epel-release centos-release-scl-rh && yum install -y wget curl which && \ wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static mysql-devel \ From efcf07a5f0a136fb2075e6af288424903c858e2c Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sat, 14 Dec 2019 12:00:56 +0800 Subject: [PATCH 17/34] fix github actions bug --- ci/docker/centos-7-core.dockerfile | 3 ++- docker/build_env/cpu/centos7/Dockerfile | 2 +- docker/build_env/gpu/centos7/Dockerfile | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/ci/docker/centos-7-core.dockerfile b/ci/docker/centos-7-core.dockerfile index b6da6a4989..ee89540564 100644 --- a/ci/docker/centos-7-core.dockerfile +++ b/ci/docker/centos-7-core.dockerfile @@ -8,11 +8,12 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN yum install -y epel-release centos-release-scl-rh && yum install -y wget curl which && \ wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static mysql-devel \ - devtoolset-6-gcc devtoolset-6-gcc-c++ devtoolset-6-gcc-gfortran llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra lcov \ + devtoolset-6-gcc devtoolset-6-gcc-c++ devtoolset-6-gcc-gfortran llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra mysql lcov \ && \ rm -rf /var/cache/yum/* RUN echo "source scl_source enable devtoolset-6" >> /etc/profile.d/devtoolset-6.sh RUN echo "source scl_source enable llvm-toolset-7.0" >> /etc/profile.d/llvm-toolset-7.sh +RUN source scl_source enable devtoolset-6 && source scl_source enable llvm-toolset-7.0 ENV CLANG_TOOLS_PATH="/opt/rh/llvm-toolset-7.0/root/usr/bin" diff --git a/docker/build_env/cpu/centos7/Dockerfile b/docker/build_env/cpu/centos7/Dockerfile index 2e564b0323..1451652d48 100644 --- a/docker/build_env/cpu/centos7/Dockerfile +++ b/docker/build_env/cpu/centos7/Dockerfile @@ -3,7 +3,7 @@ FROM centos:centos7 RUN yum install -y epel-release centos-release-scl-rh && yum install -y wget curl which && \ wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static mysql-devel \ - devtoolset-6-gcc devtoolset-6-gcc-c++ devtoolset-6-gcc-gfortran llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra lcov \ + devtoolset-6-gcc devtoolset-6-gcc-c++ devtoolset-6-gcc-gfortran llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra mysql lcov \ && \ rm -rf /var/cache/yum/* diff --git a/docker/build_env/gpu/centos7/Dockerfile b/docker/build_env/gpu/centos7/Dockerfile index b65ce1df6e..fc800d1b5d 100644 --- a/docker/build_env/gpu/centos7/Dockerfile +++ b/docker/build_env/gpu/centos7/Dockerfile @@ -5,7 +5,7 @@ ENV NVIDIA_DRIVER_CAPABILITIES compute,utility RUN yum install -y epel-release centos-release-scl-rh && yum install -y wget curl which && \ wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static mysql-devel \ - devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-gcc-gfortran llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra lcov \ + devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-gcc-gfortran llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra mysql lcov \ && \ rm -rf /var/cache/yum/* From a70ed671a766639e0e56cf07afea7a63377221b3 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sat, 14 Dec 2019 13:46:20 +0800 Subject: [PATCH 18/34] update centos7 case on github actions --- ci/docker/centos-7-core.dockerfile | 5 ++--- docker-compose.yml | 2 +- docker/build_env/cpu/centos7/Dockerfile | 4 ++-- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/ci/docker/centos-7-core.dockerfile b/ci/docker/centos-7-core.dockerfile index ee89540564..5bb7ca385f 100644 --- a/ci/docker/centos-7-core.dockerfile +++ b/ci/docker/centos-7-core.dockerfile @@ -8,12 +8,11 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] RUN yum install -y epel-release centos-release-scl-rh && yum install -y wget curl which && \ wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static mysql-devel \ - devtoolset-6-gcc devtoolset-6-gcc-c++ devtoolset-6-gcc-gfortran llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra mysql lcov \ + devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-gcc-gfortran llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra mysql lcov \ && \ rm -rf /var/cache/yum/* -RUN echo "source scl_source enable devtoolset-6" >> /etc/profile.d/devtoolset-6.sh +RUN echo "source scl_source enable devtoolset-7" >> /etc/profile.d/devtoolset-7.sh RUN echo "source scl_source enable llvm-toolset-7.0" >> /etc/profile.d/llvm-toolset-7.sh -RUN source scl_source enable devtoolset-6 && source scl_source enable llvm-toolset-7.0 ENV CLANG_TOOLS_PATH="/opt/rh/llvm-toolset-7.0/root/usr/bin" diff --git a/docker-compose.yml b/docker-compose.yml index 01adaf7292..4ee8d7aac1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -13,7 +13,7 @@ x-ccache: &ccache x-command: &cpp-command > /bin/bash -c " - /milvus/ci/scripts/build.sh -o ${MILVUS_INSTALL_PREFIX} -l -u -c + /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql" services: diff --git a/docker/build_env/cpu/centos7/Dockerfile b/docker/build_env/cpu/centos7/Dockerfile index 1451652d48..6b3817f849 100644 --- a/docker/build_env/cpu/centos7/Dockerfile +++ b/docker/build_env/cpu/centos7/Dockerfile @@ -3,11 +3,11 @@ FROM centos:centos7 RUN yum install -y epel-release centos-release-scl-rh && yum install -y wget curl which && \ wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ yum install -y ccache make automake git python3-pip libcurl-devel python3-devel boost-static mysql-devel \ - devtoolset-6-gcc devtoolset-6-gcc-c++ devtoolset-6-gcc-gfortran llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra mysql lcov \ + devtoolset-7-gcc devtoolset-7-gcc-c++ devtoolset-7-gcc-gfortran llvm-toolset-7.0-clang llvm-toolset-7.0-clang-tools-extra mysql lcov \ && \ rm -rf /var/cache/yum/* -RUN echo "source scl_source enable devtoolset-6" >> /etc/profile.d/devtoolset-6.sh +RUN echo "source scl_source enable devtoolset-7" >> /etc/profile.d/devtoolset-7.sh RUN echo "source scl_source enable llvm-toolset-7.0" >> /etc/profile.d/llvm-toolset-7.sh ENV CLANG_TOOLS_PATH="/opt/rh/llvm-toolset-7.0/root/usr/bin" From 45be63959876638a8fbf0e8241e2a53b9a747eda Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sat, 14 Dec 2019 14:01:04 +0800 Subject: [PATCH 19/34] update docker-compose.yml --- docker-compose.yml | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 4ee8d7aac1..5179ffd88b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -11,11 +11,6 @@ x-ccache: &ccache CCACHE_MAXSIZE: 2G CCACHE_DIR: /build/ccache -x-command: &cpp-command > - /bin/bash -c " - /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c - /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql" - services: db: image: mysql:5.6 @@ -43,7 +38,10 @@ services: - ${ARCH}-ubuntu-${UBUNTU}-cache:/build:delegated networks: - milvus - command: *cpp-command + command: &ubuntu-command > + /bin/bash -c " + /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c + /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql" centos-core: image: ${REPO}:${ARCH}-centos-${CENTOS}-core @@ -60,7 +58,12 @@ services: - ${ARCH}-centos-${CENTOS}-cache:/build:delegated networks: - milvus - command: *cpp-command + command: ¢os-command > + /bin/bash -c " + scl enable devtoolset-7 bash + scl enable llvm-toolset-7.0 bash + /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c + /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql" networks: milvus: From 7cc5f6dac364d89c8601fbd2cf7dfa3b2010ab49 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sat, 14 Dec 2019 14:21:41 +0800 Subject: [PATCH 20/34] debug centos case on github actions --- docker-compose.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index 5179ffd88b..39d7325e14 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -62,6 +62,10 @@ services: /bin/bash -c " scl enable devtoolset-7 bash scl enable llvm-toolset-7.0 bash + which gcc + which g++ + gcc -v + g++ -v /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql" From 1269701572dc2187fe066f1b7f7d04f064ffad4b Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sat, 14 Dec 2019 14:44:37 +0800 Subject: [PATCH 21/34] debug centos case on github actions --- docker-compose.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 39d7325e14..4eda37f488 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -60,12 +60,8 @@ services: - milvus command: ¢os-command > /bin/bash -c " - scl enable devtoolset-7 bash - scl enable llvm-toolset-7.0 bash - which gcc - which g++ - gcc -v - g++ -v + source scl_source enable devtoolset-7 + source scl_source enable llvm-toolset-7.0 /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql" From 180e1804a2965928053f980a58148b3cb26bbbd3 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Sat, 14 Dec 2019 16:04:48 +0800 Subject: [PATCH 22/34] add centos7 deploy dockerfile --- docker/deploy/cpu/centos7/Dockerfile | 17 +++++++++++++++++ .../deploy/cpu/centos7/docker-entrypoint.sh | 9 +++++++++ docker/deploy/gpu/centos7/Dockerfile | 19 +++++++++++++++++++ .../deploy/gpu/centos7/docker-entrypoint.sh | 9 +++++++++ 4 files changed, 54 insertions(+) create mode 100644 docker/deploy/cpu/centos7/Dockerfile create mode 100755 docker/deploy/cpu/centos7/docker-entrypoint.sh create mode 100644 docker/deploy/gpu/centos7/Dockerfile create mode 100755 docker/deploy/gpu/centos7/docker-entrypoint.sh diff --git a/docker/deploy/cpu/centos7/Dockerfile b/docker/deploy/cpu/centos7/Dockerfile new file mode 100644 index 0000000000..f1e4169d41 --- /dev/null +++ b/docker/deploy/cpu/centos7/Dockerfile @@ -0,0 +1,17 @@ +FROM centos:centos7 + +RUN yum install -y epel-release && \ + yum install -y libgomp libgfortran4 mysql-devel && \ + rm -rf /var/cache/yum/* + +COPY ./milvus /var/lib/milvus +COPY ./docker-entrypoint.sh /var/lib/milvus +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/var/lib/milvus/lib" + +WORKDIR /var/lib/milvus + +ENTRYPOINT [ "/var/lib/milvus/docker-entrypoint.sh" ] + +CMD [ "start" ] + +EXPOSE 19530 diff --git a/docker/deploy/cpu/centos7/docker-entrypoint.sh b/docker/deploy/cpu/centos7/docker-entrypoint.sh new file mode 100755 index 0000000000..830b394eb7 --- /dev/null +++ b/docker/deploy/cpu/centos7/docker-entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e + +if [ "$1" == 'start' ]; then + cd /var/lib/milvus/scripts && ./start_server.sh +fi + +exec "$@" diff --git a/docker/deploy/gpu/centos7/Dockerfile b/docker/deploy/gpu/centos7/Dockerfile new file mode 100644 index 0000000000..c43debfe71 --- /dev/null +++ b/docker/deploy/gpu/centos7/Dockerfile @@ -0,0 +1,19 @@ +FROM nvidia/cuda:10.1-devel-centos7 + +ENV NVIDIA_DRIVER_CAPABILITIES compute,utility + +RUN yum install -y epel-release && \ + yum install -y libgomp libgfortran4 mysql-devel && \ + rm -rf /var/cache/yum/* + +COPY ./milvus /var/lib/milvus +COPY ./docker-entrypoint.sh /var/lib/milvus +ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/var/lib/milvus/lib" + +WORKDIR /var/lib/milvus + +ENTRYPOINT [ "/var/lib/milvus/docker-entrypoint.sh" ] + +CMD [ "start" ] + +EXPOSE 19530 diff --git a/docker/deploy/gpu/centos7/docker-entrypoint.sh b/docker/deploy/gpu/centos7/docker-entrypoint.sh new file mode 100755 index 0000000000..830b394eb7 --- /dev/null +++ b/docker/deploy/gpu/centos7/docker-entrypoint.sh @@ -0,0 +1,9 @@ +#!/bin/bash + +set -e + +if [ "$1" == 'start' ]; then + cd /var/lib/milvus/scripts && ./start_server.sh +fi + +exec "$@" From ac8f47a4f9a57fe733e391b8302760f91ac32705 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 16 Dec 2019 14:12:31 +0800 Subject: [PATCH 23/34] Add CentOS 7 build case on Jenkins CI --- ci/jenkins/Jenkinsfile | 376 ++++++------------ ...vus-cpu-version-centos7-build-env-pod.yaml | 36 ++ ...pu-version-ubuntu18.04-build-env-pod.yaml} | 2 +- ...vus-gpu-version-centos7-build-env-pod.yaml | 37 ++ ...pu-version-ubuntu18.04-build-env-pod.yaml} | 2 +- 5 files changed, 203 insertions(+), 250 deletions(-) create mode 100644 ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml rename ci/jenkins/pod/{milvus-cpu-version-build-env-pod.yaml => milvus-cpu-version-ubuntu18.04-build-env-pod.yaml} (91%) create mode 100644 ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml rename ci/jenkins/pod/{milvus-gpu-version-build-env-pod.yaml => milvus-gpu-version-ubuntu18.04-build-env-pod.yaml} (92%) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 04113e65d5..02d2949e1b 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -33,275 +33,155 @@ pipeline { } stages { - stage("Ubuntu 18.04 x86_64") { - environment { - OS_NAME = "ubuntu18.04" - CPU_ARCH = "amd64" - } - - parallel { - stage ("GPU Version") { - environment { - BINRARY_VERSION = "gpu" - PACKAGE_VERSION = VersionNumber([ - versionNumberString : '${SEMVER}-gpu-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' - ]); - DOCKER_VERSION = "${SEMVER}-gpu-${OS_NAME}-${LOWER_BUILD_TYPE}" + stage ('Milvus Build and Test') { + matrix { + agent none + axes { + axis { + name 'OS_NAME' + values 'ubuntu18.04', 'centos7' } - stages { - stage("Run Build") { - agent { - kubernetes { - label "${env.BINRARY_VERSION}-build" - defaultContainer 'jnlp' - yamlFile 'ci/jenkins/pod/milvus-gpu-version-build-env-pod.yaml' - } - } + axis { + name 'CPU_ARCH' + values 'amd64' + } - stages { - stage('Build') { - steps { - container("milvus-${env.BINRARY_VERSION}-build-env") { - script { - load "${env.WORKSPACE}/ci/jenkins/step/build.groovy" - } - } - } - } - stage('Code Coverage') { - steps { - container("milvus-${env.BINRARY_VERSION}-build-env") { - script { - load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" - } - } - } - } - stage('Upload Package') { - steps { - container("milvus-${env.BINRARY_VERSION}-build-env") { - script { - load "${env.WORKSPACE}/ci/jenkins/step/package.groovy" - } - } - } - } - } - } - - stage("Publish docker images") { - agent { - kubernetes { - label "${env.BINRARY_VERSION}-publish" - defaultContainer 'jnlp' - yamlFile 'ci/jenkins/pod/docker-pod.yaml' - } - } - - stages { - stage('Publish') { - steps { - container('publish-images') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy" - } - } - } - } - } - } - - stage("Deploy to Development") { - environment { - FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-") - HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}".toLowerCase() - } - - agent { - kubernetes { - label "${env.BINRARY_VERSION}-dev-test" - defaultContainer 'jnlp' - yamlFile 'ci/jenkins/pod/testEnvironment.yaml' - } - } - - stages { - stage("Deploy to Dev") { - steps { - container('milvus-test-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/deploySingle2Dev.groovy" - } - } - } - } - - stage("Dev Test") { - steps { - container('milvus-test-env') { - script { - boolean isNightlyTest = isTimeTriggeredBuild() - if (isNightlyTest) { - load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy" - } else { - load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy" - } - } - } - } - } - - stage ("Cleanup Dev") { - steps { - container('milvus-test-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" - } - } - } - } - } - post { - unsuccessful { - container('milvus-test-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" - } - } - } - } - } + axis { + name 'BINRARY_VERSION' + values 'gpu', 'cpu' } } - - stage ("CPU Version") { - environment { - BINRARY_VERSION = "cpu" - PACKAGE_VERSION = VersionNumber([ - versionNumberString : '${SEMVER}-cpu-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' - ]); - DOCKER_VERSION = "${SEMVER}-cpu-${OS_NAME}-${LOWER_BUILD_TYPE}" - } - - stages { - stage("Run Build") { - agent { - kubernetes { - label "${env.BINRARY_VERSION}-build" - defaultContainer 'jnlp' - yamlFile 'ci/jenkins/pod/milvus-cpu-version-build-env-pod.yaml' - } - } - - stages { - stage('Build') { - steps { - container("milvus-${env.BINRARY_VERSION}-build-env") { - script { - load "${env.WORKSPACE}/ci/jenkins/step/build.groovy" - } - } - } - } - stage('Code Coverage') { - steps { - container("milvus-${env.BINRARY_VERSION}-build-env") { - script { - load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" - } - } - } - } - stage('Upload Package') { - steps { - container("milvus-${env.BINRARY_VERSION}-build-env") { - script { - load "${env.WORKSPACE}/ci/jenkins/step/package.groovy" - } - } - } - } - } + stages { + stage ("${OS_NAME} ${CPU_ARCH} ${BINRARY_VERSION}"){ + environment { + PACKAGE_VERSION = VersionNumber([ + versionNumberString : '${SEMVER}-${BINRARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' + ]); + DOCKER_VERSION = "${SEMVER}-${BINRARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}" } - stage("Publish docker images") { - agent { - kubernetes { - label "${env.BINRARY_VERSION}-publish" - defaultContainer 'jnlp' - yamlFile 'ci/jenkins/pod/docker-pod.yaml' - } - } - - stages { - stage('Publish') { - steps { - container('publish-images'){ - script { - load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy" - } - } - } - } - } - } - - stage("Deploy to Development") { - environment { - FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-") - HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${env.BINRARY_VERSION}".toLowerCase() - } - - agent { - kubernetes { - label "${env.BINRARY_VERSION}-dev-test" - defaultContainer 'jnlp' - yamlFile 'ci/jenkins/pod/testEnvironment.yaml' - } - } - - stages { - stage("Deploy to Dev") { - steps { - container('milvus-test-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/deploySingle2Dev.groovy" - } - } + stages { + stage ("Run Build") { + agent { + kubernetes { + label "${BINRARY_VERSION}-build" + defaultContainer 'jnlp' + yamlFile "ci/jenkins/pod/milvus-${BINRARY_VERSION}-version-${OS_NAME}-build-env-pod.yaml" } } - stage("Dev Test") { - steps { - container('milvus-test-env') { - script { - boolean isNightlyTest = isTimeTriggeredBuild() - if (isNightlyTest) { - load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy" - } else { - load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy" + stages { + stage ('Build') { + steps { + container("milvus-${BINRARY_VERSION}-build-env") { + script { + load "${env.WORKSPACE}/ci/jenkins/step/build.groovy" + } + } + } + } + + stage('Code Coverage') { + steps { + container("milvus-${BINRARY_VERSION}-build-env") { + script { + load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" + } + } + } + } + + stage('Upload Package') { + steps { + container("milvus-${BINRARY_VERSION}-build-env") { + script { + load "${env.WORKSPACE}/ci/jenkins/step/package.groovy" } } } } } + } - stage ("Cleanup Dev") { - steps { - container('milvus-test-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" + stage("Publish docker images") { + agent { + kubernetes { + label "${BINRARY_VERSION}-publish" + defaultContainer 'jnlp' + yamlFile 'ci/jenkins/pod/docker-pod.yaml' + } + } + + stages { + stage('Publish') { + steps { + container('publish-images') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy" + } } } } } } - post { - unsuccessful { - container('milvus-test-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" + + stage("Deploy to Development") { + environment { + FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-") + HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${BINRARY_VERSION}".toLowerCase() + } + + agent { + kubernetes { + label "${BINRARY_VERSION}-dev-test" + defaultContainer 'jnlp' + yamlFile 'ci/jenkins/pod/testEnvironment.yaml' + } + } + + stages { + stage("Deploy to Dev") { + steps { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/deploySingle2Dev.groovy" + } + } + } + } + + stage("Dev Test") { + steps { + container('milvus-test-env') { + script { + boolean isNightlyTest = isTimeTriggeredBuild() + if (isNightlyTest) { + load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy" + } else { + load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy" + } + } + } + } + } + + stage ("Cleanup Dev") { + steps { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" + } + } + } + } + } + post { + unsuccessful { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" + } } } } diff --git a/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml b/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml new file mode 100644 index 0000000000..37f9cb3598 --- /dev/null +++ b/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml @@ -0,0 +1,36 @@ +apiVersion: v1 +kind: Pod +metadata: + name: milvus-cpu-build-env + labels: + app: milvus + componet: cpu-build-env +spec: + containers: + - name: milvus-cpu-build-env + image: registry.zilliz.com/milvus/milvus-cpu-build-env:v0.7.0-centos7 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: BUILD_ENV_IMAGE_ID + value: "23476391bec80c64f10d44a6370c73c71f011a6b95114b10ff82a60e771e11c7" + command: + - cat + tty: true + resources: + limits: + memory: "12Gi" + cpu: "8.0" + requests: + memory: "8Gi" + cpu: "4.0" + - name: milvus-mysql + image: mysql:5.6 + env: + - name: MYSQL_ROOT_PASSWORD + value: 123456 + ports: + - containerPort: 3306 + name: mysql diff --git a/ci/jenkins/pod/milvus-cpu-version-build-env-pod.yaml b/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml similarity index 91% rename from ci/jenkins/pod/milvus-cpu-version-build-env-pod.yaml rename to ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml index 4631c26e09..7301be0692 100644 --- a/ci/jenkins/pod/milvus-cpu-version-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml @@ -8,7 +8,7 @@ metadata: spec: containers: - name: milvus-cpu-build-env - image: registry.zilliz.com/milvus/milvus-cpu-build-env:v0.6.0-ubuntu18.04 + image: registry.zilliz.com/milvus/milvus-cpu-build-env:v0.7.0-ubuntu18.04 env: - name: POD_IP valueFrom: diff --git a/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml b/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml new file mode 100644 index 0000000000..5bc95b7cb7 --- /dev/null +++ b/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml @@ -0,0 +1,37 @@ +apiVersion: v1 +kind: Pod +metadata: + name: milvus-gpu-build-env + labels: + app: milvus + componet: gpu-build-env +spec: + containers: + - name: milvus-gpu-build-env + image: registry.zilliz.com/milvus/milvus-gpu-build-env:v0.7.0-centos7 + env: + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + - name: BUILD_ENV_IMAGE_ID + value: "da9023b0f858f072672f86483a869aa87e90a5140864f89e5a012ec766d96dea" + command: + - cat + tty: true + resources: + limits: + memory: "12Gi" + cpu: "8.0" + nvidia.com/gpu: 1 + requests: + memory: "8Gi" + cpu: "4.0" + - name: milvus-mysql + image: mysql:5.6 + env: + - name: MYSQL_ROOT_PASSWORD + value: 123456 + ports: + - containerPort: 3306 + name: mysql diff --git a/ci/jenkins/pod/milvus-gpu-version-build-env-pod.yaml b/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml similarity index 92% rename from ci/jenkins/pod/milvus-gpu-version-build-env-pod.yaml rename to ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml index d4eff370a8..f41dda8b09 100644 --- a/ci/jenkins/pod/milvus-gpu-version-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml @@ -8,7 +8,7 @@ metadata: spec: containers: - name: milvus-gpu-build-env - image: registry.zilliz.com/milvus/milvus-gpu-build-env:v0.6.0-ubuntu18.04 + image: registry.zilliz.com/milvus/milvus-gpu-build-env:v0.7.0-ubuntu18.04 env: - name: POD_IP valueFrom: From 383f6f218c3d1069c21d0cca3f776dd16e36300e Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 16 Dec 2019 14:45:18 +0800 Subject: [PATCH 24/34] update Jenkinsfile --- ci/jenkins/Jenkinsfile | 206 ++++++++++++++++++++--------------------- 1 file changed, 103 insertions(+), 103 deletions(-) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 02d2949e1b..70de4850e1 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -52,132 +52,123 @@ pipeline { values 'gpu', 'cpu' } } - stages { - stage ("${OS_NAME} ${CPU_ARCH} ${BINRARY_VERSION}"){ - environment { - PACKAGE_VERSION = VersionNumber([ - versionNumberString : '${SEMVER}-${BINRARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' - ]); - DOCKER_VERSION = "${SEMVER}-${BINRARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}" + + + stage ("${OS_NAME} ${CPU_ARCH} ${BINRARY_VERSION}") { + environment { + PACKAGE_VERSION = VersionNumber([ + versionNumberString : '${SEMVER}-${BINRARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' + ]); + DOCKER_VERSION = "${SEMVER}-${BINRARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}" + } + + stages { + stage ("Run Build") { + agent { + kubernetes { + label "${BINRARY_VERSION}-build" + defaultContainer 'jnlp' + yamlFile "ci/jenkins/pod/milvus-${BINRARY_VERSION}-version-${OS_NAME}-build-env-pod.yaml" + } + } + + stages { + stage ('Build') { + steps { + container("milvus-${BINRARY_VERSION}-build-env") { + script { + load "${env.WORKSPACE}/ci/jenkins/step/build.groovy" + } + } + } + } + + stage('Code Coverage') { + steps { + container("milvus-${BINRARY_VERSION}-build-env") { + script { + load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" + } + } + } + } + + stage('Upload Package') { + steps { + container("milvus-${BINRARY_VERSION}-build-env") { + script { + load "${env.WORKSPACE}/ci/jenkins/step/package.groovy" + } + } + } + } + } } - stages { - stage ("Run Build") { - agent { - kubernetes { - label "${BINRARY_VERSION}-build" - defaultContainer 'jnlp' - yamlFile "ci/jenkins/pod/milvus-${BINRARY_VERSION}-version-${OS_NAME}-build-env-pod.yaml" - } + stage("Publish docker images") { + agent { + kubernetes { + label "${BINRARY_VERSION}-publish" + defaultContainer 'jnlp' + yamlFile 'ci/jenkins/pod/docker-pod.yaml' } + } - stages { - stage ('Build') { - steps { - container("milvus-${BINRARY_VERSION}-build-env") { - script { - load "${env.WORKSPACE}/ci/jenkins/step/build.groovy" - } - } - } - } - - stage('Code Coverage') { - steps { - container("milvus-${BINRARY_VERSION}-build-env") { - script { - load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" - } - } - } - } - - stage('Upload Package') { - steps { - container("milvus-${BINRARY_VERSION}-build-env") { - script { - load "${env.WORKSPACE}/ci/jenkins/step/package.groovy" - } + stages { + stage('Publish') { + steps { + container('publish-images') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy" } } } } } + } - stage("Publish docker images") { - agent { - kubernetes { - label "${BINRARY_VERSION}-publish" - defaultContainer 'jnlp' - yamlFile 'ci/jenkins/pod/docker-pod.yaml' - } - } + stage("Deploy to Development") { + environment { + FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-") + HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${BINRARY_VERSION}".toLowerCase() + } - stages { - stage('Publish') { - steps { - container('publish-images') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy" - } - } - } - } + agent { + kubernetes { + label "${BINRARY_VERSION}-dev-test" + defaultContainer 'jnlp' + yamlFile 'ci/jenkins/pod/testEnvironment.yaml' } } - stage("Deploy to Development") { - environment { - FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-") - HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${BINRARY_VERSION}".toLowerCase() - } - - agent { - kubernetes { - label "${BINRARY_VERSION}-dev-test" - defaultContainer 'jnlp' - yamlFile 'ci/jenkins/pod/testEnvironment.yaml' - } - } - - stages { - stage("Deploy to Dev") { - steps { - container('milvus-test-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/deploySingle2Dev.groovy" - } + stages { + stage("Deploy to Dev") { + steps { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/deploySingle2Dev.groovy" } } } + } - stage("Dev Test") { - steps { - container('milvus-test-env') { - script { - boolean isNightlyTest = isTimeTriggeredBuild() - if (isNightlyTest) { - load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy" - } else { - load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy" - } - } - } - } - } - - stage ("Cleanup Dev") { - steps { - container('milvus-test-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" + stage("Dev Test") { + steps { + container('milvus-test-env') { + script { + boolean isNightlyTest = isTimeTriggeredBuild() + if (isNightlyTest) { + load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy" + } else { + load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy" } } } } } - post { - unsuccessful { + + stage ("Cleanup Dev") { + steps { container('milvus-test-env') { script { load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" @@ -186,6 +177,15 @@ pipeline { } } } + post { + unsuccessful { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" + } + } + } + } } } } From bf122cc276590dfaf12536abc329f11278bef57b Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 16 Dec 2019 14:50:10 +0800 Subject: [PATCH 25/34] update Jenkinsfile --- ci/jenkins/Jenkinsfile | 227 +++++++++++++++++++++-------------------- 1 file changed, 114 insertions(+), 113 deletions(-) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 70de4850e1..6150fd3005 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -53,135 +53,136 @@ pipeline { } } - - stage ("${OS_NAME} ${CPU_ARCH} ${BINRARY_VERSION}") { - environment { - PACKAGE_VERSION = VersionNumber([ - versionNumberString : '${SEMVER}-${BINRARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' - ]); - DOCKER_VERSION = "${SEMVER}-${BINRARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}" - } - - stages { - stage ("Run Build") { - agent { - kubernetes { - label "${BINRARY_VERSION}-build" - defaultContainer 'jnlp' - yamlFile "ci/jenkins/pod/milvus-${BINRARY_VERSION}-version-${OS_NAME}-build-env-pod.yaml" - } - } - - stages { - stage ('Build') { - steps { - container("milvus-${BINRARY_VERSION}-build-env") { - script { - load "${env.WORKSPACE}/ci/jenkins/step/build.groovy" - } - } - } - } - - stage('Code Coverage') { - steps { - container("milvus-${BINRARY_VERSION}-build-env") { - script { - load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" - } - } - } - } - - stage('Upload Package') { - steps { - container("milvus-${BINRARY_VERSION}-build-env") { - script { - load "${env.WORKSPACE}/ci/jenkins/step/package.groovy" - } - } - } - } - } + stages { + stage ("${OS_NAME} ${CPU_ARCH} ${BINRARY_VERSION}") { + environment { + PACKAGE_VERSION = VersionNumber([ + versionNumberString : '${SEMVER}-${BINRARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' + ]); + DOCKER_VERSION = "${SEMVER}-${BINRARY_VERSION}-${OS_NAME}-${LOWER_BUILD_TYPE}" } - stage("Publish docker images") { - agent { - kubernetes { - label "${BINRARY_VERSION}-publish" - defaultContainer 'jnlp' - yamlFile 'ci/jenkins/pod/docker-pod.yaml' - } - } - - stages { - stage('Publish') { - steps { - container('publish-images') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy" - } - } - } - } - } - } - - stage("Deploy to Development") { - environment { - FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-") - HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${BINRARY_VERSION}".toLowerCase() - } - - agent { - kubernetes { - label "${BINRARY_VERSION}-dev-test" - defaultContainer 'jnlp' - yamlFile 'ci/jenkins/pod/testEnvironment.yaml' - } - } - - stages { - stage("Deploy to Dev") { - steps { - container('milvus-test-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/deploySingle2Dev.groovy" - } - } + stages { + stage ("Run Build") { + agent { + kubernetes { + label "${BINRARY_VERSION}-build" + defaultContainer 'jnlp' + yamlFile "ci/jenkins/pod/milvus-${BINRARY_VERSION}-version-${OS_NAME}-build-env-pod.yaml" } } - stage("Dev Test") { - steps { - container('milvus-test-env') { - script { - boolean isNightlyTest = isTimeTriggeredBuild() - if (isNightlyTest) { - load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy" - } else { - load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy" + stages { + stage ('Build') { + steps { + container("milvus-${BINRARY_VERSION}-build-env") { + script { + load "${env.WORKSPACE}/ci/jenkins/step/build.groovy" + } + } + } + } + + stage('Code Coverage') { + steps { + container("milvus-${BINRARY_VERSION}-build-env") { + script { + load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" + } + } + } + } + + stage('Upload Package') { + steps { + container("milvus-${BINRARY_VERSION}-build-env") { + script { + load "${env.WORKSPACE}/ci/jenkins/step/package.groovy" } } } } } + } - stage ("Cleanup Dev") { - steps { - container('milvus-test-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" + stage("Publish docker images") { + agent { + kubernetes { + label "${BINRARY_VERSION}-publish" + defaultContainer 'jnlp' + yamlFile 'ci/jenkins/pod/docker-pod.yaml' + } + } + + stages { + stage('Publish') { + steps { + container('publish-images') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/publishImages.groovy" + } } } } } } - post { - unsuccessful { - container('milvus-test-env') { - script { - load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" + + stage("Deploy to Development") { + environment { + FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-") + HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${BINRARY_VERSION}".toLowerCase() + } + + agent { + kubernetes { + label "${BINRARY_VERSION}-dev-test" + defaultContainer 'jnlp' + yamlFile 'ci/jenkins/pod/testEnvironment.yaml' + } + } + + stages { + stage("Deploy to Dev") { + steps { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/deploySingle2Dev.groovy" + } + } + } + } + + stage("Dev Test") { + steps { + container('milvus-test-env') { + script { + boolean isNightlyTest = isTimeTriggeredBuild() + if (isNightlyTest) { + load "${env.WORKSPACE}/ci/jenkins/step/singleDevNightlyTest.groovy" + } else { + load "${env.WORKSPACE}/ci/jenkins/step/singleDevTest.groovy" + } + } + } + } + } + + stage ("Cleanup Dev") { + steps { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" + } + } + } + } + } + post { + unsuccessful { + container('milvus-test-env') { + script { + load "${env.WORKSPACE}/ci/jenkins/step/cleanupSingleDev.groovy" + } } } } From 6535e607c4ff4118677bfe4983d6bb93e1074ad6 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 16 Dec 2019 15:41:59 +0800 Subject: [PATCH 26/34] update Jenkins CI --- ci/jenkins/Jenkinsfile | 16 ++++++++-------- ...milvus-cpu-version-centos7-build-env-pod.yaml | 4 +++- ...us-cpu-version-ubuntu18.04-build-env-pod.yaml | 2 ++ ...milvus-gpu-version-centos7-build-env-pod.yaml | 4 +++- ...us-gpu-version-ubuntu18.04-build-env-pod.yaml | 2 ++ ci/jenkins/step/build.groovy | 2 +- ci/jenkins/step/deploySingle2Dev.groovy | 2 +- ci/jenkins/step/publishImages.groovy | 2 +- ci/jenkins/step/singleDevNightlyTest.groovy | 2 +- ci/jenkins/step/singleDevTest.groovy | 2 +- ci/scripts/check_ccache.sh | 14 +++++++------- ci/scripts/update_ccache.sh | 5 ++--- 12 files changed, 32 insertions(+), 25 deletions(-) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 6150fd3005..697d7592e7 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -54,7 +54,7 @@ pipeline { } stages { - stage ("${OS_NAME} ${CPU_ARCH} ${BINRARY_VERSION}") { + stage("Milvus ${OS_NAME} ${CPU_ARCH} ${BINRARY_VERSION}") { environment { PACKAGE_VERSION = VersionNumber([ versionNumberString : '${SEMVER}-${BINRARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' @@ -63,7 +63,7 @@ pipeline { } stages { - stage ("Run Build") { + stage('Run Build') { agent { kubernetes { label "${BINRARY_VERSION}-build" @@ -73,7 +73,7 @@ pipeline { } stages { - stage ('Build') { + stage('Build') { steps { container("milvus-${BINRARY_VERSION}-build-env") { script { @@ -105,7 +105,7 @@ pipeline { } } - stage("Publish docker images") { + stage('Publish docker images') { agent { kubernetes { label "${BINRARY_VERSION}-publish" @@ -127,7 +127,7 @@ pipeline { } } - stage("Deploy to Development") { + stage('Deploy to Development') { environment { FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-") HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${BINRARY_VERSION}".toLowerCase() @@ -142,7 +142,7 @@ pipeline { } stages { - stage("Deploy to Dev") { + stage('Deploy to Dev') { steps { container('milvus-test-env') { script { @@ -152,7 +152,7 @@ pipeline { } } - stage("Dev Test") { + stage('Dev Test') { steps { container('milvus-test-env') { script { @@ -167,7 +167,7 @@ pipeline { } } - stage ("Cleanup Dev") { + stage('Cleanup Dev') { steps { container('milvus-test-env') { script { diff --git a/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml b/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml index 37f9cb3598..6091a01eeb 100644 --- a/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-cpu-version-centos7-build-env-pod.yaml @@ -14,8 +14,10 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + - name: OS_NAME + value: "centos7" - name: BUILD_ENV_IMAGE_ID - value: "23476391bec80c64f10d44a6370c73c71f011a6b95114b10ff82a60e771e11c7" + value: "225b4d9c26d67b70b476964b4dd6e216de4b464d7a973a8c0c7ed1313c4d81ad" command: - cat tty: true diff --git a/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml b/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml index 7301be0692..4689da5db2 100644 --- a/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-cpu-version-ubuntu18.04-build-env-pod.yaml @@ -14,6 +14,8 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + - name: OS_NAME + value: "ubuntu18.04" - name: BUILD_ENV_IMAGE_ID value: "23476391bec80c64f10d44a6370c73c71f011a6b95114b10ff82a60e771e11c7" command: diff --git a/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml b/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml index 5bc95b7cb7..34da55a93d 100644 --- a/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-gpu-version-centos7-build-env-pod.yaml @@ -14,8 +14,10 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + - name: OS_NAME + value: "centos7" - name: BUILD_ENV_IMAGE_ID - value: "da9023b0f858f072672f86483a869aa87e90a5140864f89e5a012ec766d96dea" + value: "a5ec9914737ea4727d88ae36b4a73ca5d817f19438ba913cc1de6a1ee2ed2336" command: - cat tty: true diff --git a/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml b/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml index f41dda8b09..95b47c9111 100644 --- a/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml +++ b/ci/jenkins/pod/milvus-gpu-version-ubuntu18.04-build-env-pod.yaml @@ -14,6 +14,8 @@ spec: valueFrom: fieldRef: fieldPath: status.podIP + - name: OS_NAME + value: "ubuntu18.04" - name: BUILD_ENV_IMAGE_ID value: "da9023b0f858f072672f86483a869aa87e90a5140864f89e5a012ec766d96dea" command: diff --git a/ci/jenkins/step/build.groovy b/ci/jenkins/step/build.groovy index 05924f7308..ef8ce5be90 100644 --- a/ci/jenkins/step/build.groovy +++ b/ci/jenkins/step/build.groovy @@ -2,7 +2,7 @@ timeout(time: 60, unit: 'MINUTES') { dir ("ci/scripts") { withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { def checkResult = sh(script: "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache", returnStatus: true) - if ("${env.BINRARY_VERSION}" == "gpu") { + if ("${BINRARY_VERSION}" == "gpu") { sh ". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -g -u -c" } else { sh ". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -u -c" diff --git a/ci/jenkins/step/deploySingle2Dev.groovy b/ci/jenkins/step/deploySingle2Dev.groovy index cb2ad2b1cb..e566c3da0c 100644 --- a/ci/jenkins/step/deploySingle2Dev.groovy +++ b/ci/jenkins/step/deploySingle2Dev.groovy @@ -3,7 +3,7 @@ sh 'helm repo update' dir ('milvus-helm') { checkout([$class: 'GitSCM', branches: [[name: "0.6.0"]], userRemoteConfigs: [[url: "https://github.com/milvus-io/milvus-helm.git", name: 'origin', refspec: "+refs/heads/0.6.0:refs/remotes/origin/0.6.0"]]]) dir ("milvus") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/sqlite_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/sqlite_${BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } diff --git a/ci/jenkins/step/publishImages.groovy b/ci/jenkins/step/publishImages.groovy index 5449bcedd8..fe1c211abc 100644 --- a/ci/jenkins/step/publishImages.groovy +++ b/ci/jenkins/step/publishImages.groovy @@ -1,5 +1,5 @@ timeout(time: 15, unit: 'MINUTES') { - dir ("docker/deploy/${env.BINRARY_VERSION}/${env.OS_NAME}") { + dir ("docker/deploy/${BINRARY_VERSION}/${OS_NAME}") { def binaryPackage = "${PROJECT_NAME}-${PACKAGE_VERSION}.tar.gz" withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'JFROG_USERNAME', passwordVariable: 'JFROG_PASSWORD')]) { diff --git a/ci/jenkins/step/singleDevNightlyTest.groovy b/ci/jenkins/step/singleDevNightlyTest.groovy index d14ba1b66c..618c2457b3 100644 --- a/ci/jenkins/step/singleDevNightlyTest.groovy +++ b/ci/jenkins/step/singleDevNightlyTest.groovy @@ -13,7 +13,7 @@ timeout(time: 90, unit: 'MINUTES') { } dir ("milvus-helm") { dir ("milvus") { - sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." } } dir ("tests/milvus_python_test") { diff --git a/ci/jenkins/step/singleDevTest.groovy b/ci/jenkins/step/singleDevTest.groovy index 7b72eaacde..255f748996 100644 --- a/ci/jenkins/step/singleDevTest.groovy +++ b/ci/jenkins/step/singleDevTest.groovy @@ -14,7 +14,7 @@ timeout(time: 60, unit: 'MINUTES') { // } // dir ("milvus-helm") { // dir ("milvus") { - // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${env.BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." + // sh "helm install --wait --timeout 300 --set engine.image.tag=${DOCKER_VERSION} --set expose.type=clusterIP --name ${env.HELM_RELEASE_NAME} -f ci/db_backend/mysql_${BINRARY_VERSION}_values.yaml -f ci/filebeat/values.yaml --namespace milvus ." // } // } // dir ("tests/milvus_python_test") { diff --git a/ci/scripts/check_ccache.sh b/ci/scripts/check_ccache.sh index 2350bd558d..cc0708505e 100755 --- a/ci/scripts/check_ccache.sh +++ b/ci/scripts/check_ccache.sh @@ -1,7 +1,6 @@ #!/bin/bash -OS_NAME="linux" -CODE_NAME=$(lsb_release -sc) +OS_NAME="${OS_NAME}" BUILD_ENV_DOCKER_IMAGE_ID="${BUILD_ENV_IMAGE_ID}" BRANCH_NAMES=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's=[a-zA-Z]*\/==g' | awk -F", " '{$1=""; print $0}') ARTIFACTORY_URL="" @@ -43,13 +42,14 @@ fi check_ccache() { BRANCH=$1 - echo "fetching ${BRANCH}/ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" - wget -q --method HEAD "${ARTIFACTORY_URL}/${BRANCH}/ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" + PACKAGE_FILE="ccache-${OS_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" + echo "fetching ${BRANCH}/${PACKAGE_FILE}" + wget -q --method HEAD "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" if [[ $? == 0 ]];then - wget -q "${ARTIFACTORY_URL}/${BRANCH}/ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" && \ + wget -q "${ARTIFACTORY_URL}/${BRANCH}/${PACKAGE_FILE}" && \ mkdir -p ${CCACHE_DIRECTORY} && \ - tar zxf ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz -C ${CCACHE_DIRECTORY} && \ - rm ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz + tar zxf ${PACKAGE_FILE} -C ${CCACHE_DIRECTORY} && \ + rm ${PACKAGE_FILE} if [[ $? == 0 ]];then echo "found cache" exit 0 diff --git a/ci/scripts/update_ccache.sh b/ci/scripts/update_ccache.sh index f4afc29d1e..ec52bd1156 100755 --- a/ci/scripts/update_ccache.sh +++ b/ci/scripts/update_ccache.sh @@ -1,7 +1,6 @@ #!/bin/bash -OS_NAME="linux" -CODE_NAME=$(lsb_release -sc) +OS_NAME="${OS_NAME}" BUILD_ENV_DOCKER_IMAGE_ID="${BUILD_ENV_IMAGE_ID}" BRANCH_NAME=$(git log --decorate | head -n 1 | sed 's/.*(\(.*\))/\1/' | sed 's/.*, //' | sed 's=[a-zA-Z]*\/==g') ARTIFACTORY_URL="" @@ -51,7 +50,7 @@ if [[ -z "${ARTIFACTORY_URL}" || "${ARTIFACTORY_URL}" == "" ]];then exit 1 fi -PACKAGE_FILE="ccache-${OS_NAME}-${CODE_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" +PACKAGE_FILE="ccache-${OS_NAME}-${BUILD_ENV_DOCKER_IMAGE_ID}.tar.gz" REMOTE_PACKAGE_PATH="${ARTIFACTORY_URL}/${BRANCH_NAME}" ccache --show-stats From 0c68f59d40fb175d816395d6d6f3bfd2967c2edd Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 16 Dec 2019 15:43:46 +0800 Subject: [PATCH 27/34] update Jenkins CI --- ci/jenkins/Jenkinsfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index 697d7592e7..a98a3df336 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -54,7 +54,7 @@ pipeline { } stages { - stage("Milvus ${OS_NAME} ${CPU_ARCH} ${BINRARY_VERSION}") { + stage("Milvus CI") { environment { PACKAGE_VERSION = VersionNumber([ versionNumberString : '${SEMVER}-${BINRARY_VERSION}-${OS_NAME}-${CPU_ARCH}-${LOWER_BUILD_TYPE}-${BUILD_DATE_FORMATTED, "yyyyMMdd"}-${BUILDS_TODAY}' From fa91ff32c975c2a980be3e9ff884c9f981baeacd Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 16 Dec 2019 16:40:58 +0800 Subject: [PATCH 28/34] update Jenkins CI --- .github/workflows/core.yml | 4 ++-- ci/jenkins/Jenkinsfile | 14 +++++++++----- ci/scripts/coverage.sh | 19 +++++++++++++------ docker-compose.yml | 4 ++-- 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index c1c09e01df..d571a4eb33 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -31,7 +31,7 @@ jobs: - name: Docker Build run: | docker-compose run --use-aliases -d db - docker-compose run ubuntu-core + docker-compose run -e CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} ubuntu-core - name: Docker Push if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' continue-on-error: true @@ -61,7 +61,7 @@ jobs: - name: Docker Build run: | docker-compose run --use-aliases -d db - docker-compose run centos-core + docker-compose run -e CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} centos-core - name: Docker Push if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' continue-on-error: true diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index a98a3df336..c3e66baa92 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -66,7 +66,7 @@ pipeline { stage('Run Build') { agent { kubernetes { - label "${BINRARY_VERSION}-build" + label "${OS_NAME}-${BINRARY_VERSION}-build" defaultContainer 'jnlp' yamlFile "ci/jenkins/pod/milvus-${BINRARY_VERSION}-version-${OS_NAME}-build-env-pod.yaml" } @@ -87,7 +87,11 @@ pipeline { steps { container("milvus-${BINRARY_VERSION}-build-env") { script { - load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" + if ("${BINRARY_VERSION}" == "gpu") { + load "${env.WORKSPACE}/ci/jenkins/step/coverage.groovy" + } else { + echo "Skip Code Coverage" + } } } } @@ -108,7 +112,7 @@ pipeline { stage('Publish docker images') { agent { kubernetes { - label "${BINRARY_VERSION}-publish" + label "${OS_NAME}-${BINRARY_VERSION}-publish" defaultContainer 'jnlp' yamlFile 'ci/jenkins/pod/docker-pod.yaml' } @@ -130,12 +134,12 @@ pipeline { stage('Deploy to Development') { environment { FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-") - HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${BINRARY_VERSION}".toLowerCase() + HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${OS_NAME}-${BINRARY_VERSION}".toLowerCase() } agent { kubernetes { - label "${BINRARY_VERSION}-dev-test" + label "${OS_NAME}-${BINRARY_VERSION}-dev-test" defaultContainer 'jnlp' yamlFile 'ci/jenkins/pod/testEnvironment.yaml' } diff --git a/ci/scripts/coverage.sh b/ci/scripts/coverage.sh index 2df0cdbeb0..2cd502870f 100755 --- a/ci/scripts/coverage.sh +++ b/ci/scripts/coverage.sh @@ -13,10 +13,11 @@ MILVUS_CORE_DIR="${SCRIPTS_DIR}/../../core" CORE_BUILD_DIR="${MILVUS_CORE_DIR}/cmake_build" MYSQL_USER_NAME=root MYSQL_PASSWORD=123456 -MYSQL_HOST='127.0.0.1' -MYSQL_PORT='3306' +MYSQL_HOST="127.0.0.1" +MYSQL_PORT="3306" +CODECOV_TOKEN="" -while getopts "o:b:u:p:t:h" arg +while getopts "o:b:u:p:t:c:h" arg do case $arg in o) @@ -34,6 +35,9 @@ do t) MYSQL_HOST=$OPTARG ;; + c) + CODECOV_TOKEN=$OPTARG + ;; h) # help echo " @@ -43,10 +47,11 @@ parameter: -u: mysql account -p: mysql password -t: mysql host +-c: codecov token -h: help usage: -./coverage.sh -o \${INSTALL_PREFIX} -b \${CORE_BUILD_DIR} -u \${MYSQL_USER} -p \${MYSQL_PASSWORD} -t \${MYSQL_HOST} [-h] +./coverage.sh -o \${INSTALL_PREFIX} -b \${CORE_BUILD_DIR} -u \${MYSQL_USER} -p \${MYSQL_PASSWORD} -t \${MYSQL_HOST} -c \${CODECOV_TOKEN} [-h] " exit 0 ;; @@ -146,5 +151,7 @@ if [ $? -ne 0 ]; then exit 2 fi -# gen html report -# ${LCOV_GEN_CMD} "${FILE_INFO_OUTPUT_NEW}" --output-directory ${DIR_LCOV_OUTPUT}/ +if [[ ! -z ${CODECOV_TOKEN} ]];then + export CODECOV_TOKEN="${CODECOV_TOKEN}" + curl -s https://codecov.io/bash | bash -s - -f output_new.info || echo "Codecov did not collect coverage reports" +fi diff --git a/docker-compose.yml b/docker-compose.yml index 4eda37f488..d47b223442 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,7 +41,7 @@ services: command: &ubuntu-command > /bin/bash -c " /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c - /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql" + /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql -c ${CODECOV_TOKEN}" centos-core: image: ${REPO}:${ARCH}-centos-${CENTOS}-core @@ -63,7 +63,7 @@ services: source scl_source enable devtoolset-7 source scl_source enable llvm-toolset-7.0 /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c - /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql" + /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql -c ${CODECOV_TOKEN}" networks: milvus: From 65990ea3e679fc9570416db16d0df4138e36a502 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 16 Dec 2019 17:09:58 +0800 Subject: [PATCH 29/34] update Jenkins CI --- ci/jenkins/step/build.groovy | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/ci/jenkins/step/build.groovy b/ci/jenkins/step/build.groovy index ef8ce5be90..71643bef51 100644 --- a/ci/jenkins/step/build.groovy +++ b/ci/jenkins/step/build.groovy @@ -3,9 +3,17 @@ timeout(time: 60, unit: 'MINUTES') { withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { def checkResult = sh(script: "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache", returnStatus: true) if ("${BINRARY_VERSION}" == "gpu") { - sh ". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -g -u -c" + if ("${OS_NAME}" == "centos7") { + sh ". ./before-install.sh && source scl_source enable devtoolset-7 && source scl_source enable llvm-toolset-7.0 && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -g -u -c" + } else { + sh ". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -g -u -c" + } } else { - sh ". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -u -c" + if ("${OS_NAME}" == "centos7") { + sh ". ./before-install.sh && source scl_source enable devtoolset-7 && source scl_source enable llvm-toolset-7.0 && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -u -c" + } else { + sh ". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -u -c" + } } sh "./update_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache -u ${USERNAME} -p ${PASSWORD}" } From 2baf460e1bcf4919abe7752f88b48e78a042c5db Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 16 Dec 2019 17:44:16 +0800 Subject: [PATCH 30/34] update Jenkins CI --- .github/workflows/core.yml | 4 ++-- docker-compose.yml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/core.yml b/.github/workflows/core.yml index d571a4eb33..c1c09e01df 100644 --- a/.github/workflows/core.yml +++ b/.github/workflows/core.yml @@ -31,7 +31,7 @@ jobs: - name: Docker Build run: | docker-compose run --use-aliases -d db - docker-compose run -e CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} ubuntu-core + docker-compose run ubuntu-core - name: Docker Push if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' continue-on-error: true @@ -61,7 +61,7 @@ jobs: - name: Docker Build run: | docker-compose run --use-aliases -d db - docker-compose run -e CODECOV_TOKEN=${{ secrets.CODECOV_TOKEN }} centos-core + docker-compose run centos-core - name: Docker Push if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' continue-on-error: true diff --git a/docker-compose.yml b/docker-compose.yml index d47b223442..4eda37f488 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -41,7 +41,7 @@ services: command: &ubuntu-command > /bin/bash -c " /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c - /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql -c ${CODECOV_TOKEN}" + /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql" centos-core: image: ${REPO}:${ARCH}-centos-${CENTOS}-core @@ -63,7 +63,7 @@ services: source scl_source enable devtoolset-7 source scl_source enable llvm-toolset-7.0 /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c - /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql -c ${CODECOV_TOKEN}" + /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql" networks: milvus: From f45a96500f49dd048ce92c6cfe440701818c5275 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Mon, 16 Dec 2019 18:31:15 +0800 Subject: [PATCH 31/34] update Jenkins CI --- ci/jenkins/Jenkinsfile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/jenkins/Jenkinsfile b/ci/jenkins/Jenkinsfile index c3e66baa92..12481eae12 100644 --- a/ci/jenkins/Jenkinsfile +++ b/ci/jenkins/Jenkinsfile @@ -134,7 +134,8 @@ pipeline { stage('Deploy to Development') { environment { FROMAT_SEMVER = "${env.SEMVER}".replaceAll("\\.", "-") - HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${OS_NAME}-${BINRARY_VERSION}".toLowerCase() + FORMAT_OS_NAME = "${OS_NAME}".replaceAll("\\.", "-") + HELM_RELEASE_NAME = "${env.PIPELINE_NAME}-${env.FROMAT_SEMVER}-${env.BUILD_NUMBER}-single-${FORMAT_OS_NAME}-${BINRARY_VERSION}".toLowerCase() } agent { From a013d6bbc900006d6d331057632edf1f28b06377 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Tue, 17 Dec 2019 11:45:31 +0800 Subject: [PATCH 32/34] update Jenkins CI --- ci/jenkins/step/build.groovy | 12 ++---------- ci/scripts/base.info | 0 ci/scripts/server.info | 0 docker-compose.yml | 4 +--- 4 files changed, 3 insertions(+), 13 deletions(-) create mode 100644 ci/scripts/base.info create mode 100644 ci/scripts/server.info diff --git a/ci/jenkins/step/build.groovy b/ci/jenkins/step/build.groovy index 71643bef51..9034c936ec 100644 --- a/ci/jenkins/step/build.groovy +++ b/ci/jenkins/step/build.groovy @@ -3,17 +3,9 @@ timeout(time: 60, unit: 'MINUTES') { withCredentials([usernamePassword(credentialsId: "${params.JFROG_CREDENTIALS_ID}", usernameVariable: 'USERNAME', passwordVariable: 'PASSWORD')]) { def checkResult = sh(script: "./check_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache", returnStatus: true) if ("${BINRARY_VERSION}" == "gpu") { - if ("${OS_NAME}" == "centos7") { - sh ". ./before-install.sh && source scl_source enable devtoolset-7 && source scl_source enable llvm-toolset-7.0 && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -g -u -c" - } else { - sh ". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -g -u -c" - } + sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -g -u -c\"" } else { - if ("${OS_NAME}" == "centos7") { - sh ". ./before-install.sh && source scl_source enable devtoolset-7 && source scl_source enable llvm-toolset-7.0 && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -u -c" - } else { - sh ". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -u -c" - } + sh "/bin/bash --login -c \". ./before-install.sh && ./build.sh -t ${params.BUILD_TYPE} -o ${env.MILVUS_INSTALL_PREFIX} -l -u -c\"" } sh "./update_ccache.sh -l ${params.JFROG_ARTFACTORY_URL}/ccache -u ${USERNAME} -p ${PASSWORD}" } diff --git a/ci/scripts/base.info b/ci/scripts/base.info new file mode 100644 index 0000000000..e69de29bb2 diff --git a/ci/scripts/server.info b/ci/scripts/server.info new file mode 100644 index 0000000000..e69de29bb2 diff --git a/docker-compose.yml b/docker-compose.yml index 4eda37f488..59b3e7ce46 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -59,9 +59,7 @@ services: networks: - milvus command: ¢os-command > - /bin/bash -c " - source scl_source enable devtoolset-7 - source scl_source enable llvm-toolset-7.0 + /bin/bash --login -c " /milvus/ci/scripts/build.sh -t Release -o ${MILVUS_INSTALL_PREFIX} -l -u -c /milvus/ci/scripts/coverage.sh -o ${MILVUS_INSTALL_PREFIX} -u root -p 123456 -t mysql" From 6ee32c9adb26eedb8cf84880eb340be9e1370442 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Tue, 17 Dec 2019 11:46:11 +0800 Subject: [PATCH 33/34] update Jenkins CI --- ci/scripts/base.info | 0 ci/scripts/server.info | 0 2 files changed, 0 insertions(+), 0 deletions(-) delete mode 100644 ci/scripts/base.info delete mode 100644 ci/scripts/server.info diff --git a/ci/scripts/base.info b/ci/scripts/base.info deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/ci/scripts/server.info b/ci/scripts/server.info deleted file mode 100644 index e69de29bb2..0000000000 From c32f58d474f0ddac8bce012cc311eaae1de3adf3 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Tue, 17 Dec 2019 16:45:48 +0800 Subject: [PATCH 34/34] update GRPC url --- core/cmake/ThirdPartyPackages.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/cmake/ThirdPartyPackages.cmake b/core/cmake/ThirdPartyPackages.cmake index 0cca25ebba..8d767f8200 100644 --- a/core/cmake/ThirdPartyPackages.cmake +++ b/core/cmake/ThirdPartyPackages.cmake @@ -331,7 +331,7 @@ if (DEFINED ENV{MILVUS_GRPC_URL}) set(GRPC_SOURCE_URL "$ENV{MILVUS_GRPC_URL}") else () set(GRPC_SOURCE_URL - "https://github.com/youny626/grpc-milvus/archive/${GRPC_VERSION}.zip") + "https://github.com/ZhifengZhang-CN/grpc-milvus/archive/${GRPC_VERSION}.zip") endif () set(GRPC_MD5 "0362ba219f59432c530070b5f5c3df73")