diff --git a/.env b/.env index 9dc6a687ea..0cf22c3715 100644 --- a/.env +++ b/.env @@ -1,10 +1,10 @@ IMAGE_REPO=milvusdb IMAGE_ARCH=amd64 OS_NAME=ubuntu20.04 -DATE_VERSION=20230320-f547c1f -LATEST_DATE_VERSION=20230320-f547c1f +DATE_VERSION=20230818-533f0dd +LATEST_DATE_VERSION=20230818-533f0dd GPU_DATE_VERSION=20230317-a1c7b0c LATEST_GPU_DATE_VERSION=20230317-a1c7b0c MINIO_ADDRESS=minio:9000 PULSAR_ADDRESS=pulsar://pulsar:6650 -ETCD_ENDPOINTS=etcd:2379 \ No newline at end of file +ETCD_ENDPOINTS=etcd:2379 diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index 3b7fb572e2..8606a20d5c 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -52,7 +52,7 @@ All Linux distributions are available for Milvus development. However a majority Here's a list of verified OS types where Milvus can successfully build and run: - Debian/Ubuntu -- CentOS +- Amazon Linux - MacOS (x86_64) - MacOS (Apple Silicon) diff --git a/build/README.md b/build/README.md index 42b23f42ed..1801e1a8d5 100644 --- a/build/README.md +++ b/build/README.md @@ -66,12 +66,12 @@ The following scripts are found in the [`build/`](.) directory. Note that all sc - `build/builder.sh make unittest`: Run all unit tests. - `build/builder.sh make clean`: Clean up all the generated files. -You can specify different OS for builder by setting `OS_NAME` which defaults to `ubuntu20.04`. Valid OS are `ubuntu20.04`, `centos7`. +You can specify different OS for builder by setting `OS_NAME` which defaults to `ubuntu20.04`. Valid OS are `ubuntu20.04`, `amazonlinux2023`. -To specify `centos7` builder, use these commands: +To specify `amazonlinux2023` builder, use these commands: ```shell -export OS_NAME=centos7 +export OS_NAME=amazonlinux2023 build/builder.sh make ``` diff --git a/build/builder.sh b/build/builder.sh index f0086dfa19..4c7cbbf081 100755 --- a/build/builder.sh +++ b/build/builder.sh @@ -30,7 +30,7 @@ mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-conan" chmod -R 777 "${DOCKER_VOLUME_DIRECTORY:-.docker}" docker-compose pull --ignore-pull-failures builder -if [[ "${CHECK_BUILDER:-1}" == "1" ]]; then +if [[ "${CHECK_BUILDER:-}" == "1" ]]; then docker-compose build builder fi diff --git a/build/docker/builder/cpu/centos7/Dockerfile b/build/docker/builder/cpu/centos7/Dockerfile deleted file mode 100644 index 640899d9ee..0000000000 --- a/build/docker/builder/cpu/centos7/Dockerfile +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright (C) 2019-2020 Zilliz. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under the License. - -FROM milvusdb/openblas:centos7-20230215 - -RUN yum install -y epel-release centos-release-scl-rh && yum install -y wget curl which && \ - wget -qO- "https://cmake.org/files/v3.24/cmake-3.24.0-linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ - yum install -y git make automake python3-devel \ - devtoolset-11-gcc devtoolset-11-gcc-c++ devtoolset-11-gcc-gfortran \ - llvm-toolset-11.0-clang llvm-toolset-11.0-clang-tools-extra \ - libaio libuuid-devel unzip \ - ccache lcov libtool m4 autoconf automake && \ - rm -rf /var/cache/yum/* && \ - echo "source scl_source enable devtoolset-11" >> /etc/profile.d/devtoolset-11.sh && \ - echo "source scl_source enable llvm-toolset-11.0" >> /etc/profile.d/llvm-toolset-11.sh - -RUN pip3 install conan==1.58.0 - -RUN mkdir /tmp/stdlib && cd /tmp/stdlib && \ - wget https://repo.anaconda.com/archive/Anaconda3-2022.10-Linux-x86_64.sh && \ - sh Anaconda3-2022.10-Linux-x86_64.sh -b -p conda && \ - cp conda/lib/libstdc++.so.6.0.29 /usr/lib64 && \ - rm /usr/lib64/libstdc++.so.6 && \ - ln -s /usr/lib64/libstdc++.so.6.0.29 /usr/lib64/libstdc++.so.6 && \ - cp conda/lib/libatomic.so.1.2.0 /usr/lib64 && \ - ln -s /usr/lib64/libatomic.so.1.2.0 /usr/lib64/libatomic.so && \ - ln -s /usr/lib64/libatomic.so.1.2.0 /usr/lib64/libatomic.so.1 && \ - rm -rf /tmp/stdlib - -ENV CLANG_TOOLS_PATH="/opt/rh/llvm-toolset-11.0/root/usr/bin" -ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH - -# Install Go -ENV GOPATH /go -ENV GOROOT /usr/local/go -ENV GO111MODULE on -ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH -RUN mkdir -p /usr/local/go && wget -qO- "https://golang.org/dl/go1.20.4.linux-amd64.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \ - mkdir -p "$GOPATH/src" "$GOPATH/bin" && \ - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ${GOPATH}/bin v1.46.2 && \ - go install github.com/ramya-rao-a/go-outline@latest && \ - go install golang.org/x/tools/gopls@latest && \ - go install github.com/uudashr/gopkgs/v2/cmd/gopkgs@latest && \ - go install github.com/go-delve/delve/cmd/dlv@latest && \ - go install honnef.co/go/tools/cmd/staticcheck@2022.1 && \ - go clean --modcache && \ - chmod -R 777 "$GOPATH" && chmod -R a+w $(go env GOTOOLDIR) - -RUN ln -s /go/bin/dlv /go/bin/dlv-dap - -RUN echo 'root:root' | chpasswd - -# refer: https://code.visualstudio.com/docs/remote/containers-advanced#_avoiding-extension-reinstalls-on-container-rebuild -RUN mkdir -p /home/milvus/.vscode-server/extensions \ - /home/milvus/.vscode-server-insiders/extensions \ - && chmod -R 777 /home/milvus - -COPY --chown=0:0 build/docker/builder/entrypoint.sh / - -RUN wget -qO- "https://github.com/jeffoverflow/autouseradd/releases/download/1.2.0/autouseradd-1.2.0-amd64.tar.gz" | tar xz -C / --strip-components 1 - -RUN wget -O /tini https://github.com/krallin/tini/releases/download/v0.19.0/tini && \ - chmod +x /tini - -ENTRYPOINT [ "/tini", "--", "autouseradd", "--user", "milvus", "--", "/entrypoint.sh" ] -CMD ["tail", "-f", "/dev/null"] diff --git a/build/docker/milvus/centos7/Dockerfile b/build/docker/milvus/centos7/Dockerfile deleted file mode 100644 index c24af688da..0000000000 --- a/build/docker/milvus/centos7/Dockerfile +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright (C) 2019-2020 Zilliz. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under the License. - -FROM milvusdb/openblas:centos7-20230215 AS openblas - -FROM centos:centos7 - -RUN yum install -y wget && \ - wget -P /etc/yum.repos.d/ http://mirrors.aliyun.com/repo/epel-7.repo && \ - yum clean all && yum makecache && \ - yum install -y libgomp libgfortran4 tbb-devel libzstd-devel libaio && \ - rm -rf /var/cache/yum/* - -COPY --from=openblas /usr/lib/libopenblas-r0.3.9.so /usr/lib/ - -RUN ln -s /usr/lib/libopenblas-r0.3.9.so /usr/lib/libopenblas.so.0 && \ - ln -s /usr/lib/libopenblas.so.0 /usr/lib/libopenblas.so - -COPY ./bin/ /milvus/bin/ - -COPY ./configs/ /milvus/configs/ - -COPY ./lib/ /milvus/lib/ - -ENV PATH=/milvus/bin:$PATH -ENV LD_LIBRARY_PATH=/milvus/lib:$LD_LIBRARY_PATH:/usr/lib -ENV LD_PRELOAD=/milvus/lib/libjemalloc.so - -# Add Tini -ADD https://github.com/krallin/tini/releases/download/v0.19.0/tini /tini -RUN chmod +x /tini -ENTRYPOINT ["/tini", "--"] - -WORKDIR /milvus diff --git a/build/docker/openblas/centos7/Dockerfile b/build/docker/openblas/centos7/Dockerfile deleted file mode 100644 index 0394a5cd3d..0000000000 --- a/build/docker/openblas/centos7/Dockerfile +++ /dev/null @@ -1,17 +0,0 @@ -FROM centos:centos7 - -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -RUN yum install -y epel-release centos-release-scl-rh && yum install -y wget make automake \ - devtoolset-11-gcc devtoolset-11-gcc-c++ devtoolset-11-gcc-gfortran && \ - rm -rf /var/cache/yum/* && \ - echo "source scl_source enable devtoolset-11" >> /etc/profile.d/devtoolset-11.sh - -RUN source /etc/profile.d/devtoolset-11.sh && \ - wget https://github.com/xianyi/OpenBLAS/archive/v0.3.9.tar.gz && \ - tar zxvf v0.3.9.tar.gz && cd OpenBLAS-0.3.9 && \ - make TARGET=CORE2 DYNAMIC_ARCH=1 DYNAMIC_OLDER=1 USE_THREAD=0 USE_OPENMP=0 FC=gfortran CC=gcc COMMON_OPT="-O3 -g -fPIC" FCOMMON_OPT="-O3 -g -fPIC -frecursive" NMAX="NUM_THREADS=128" LIBPREFIX="libopenblas" INTERFACE64=0 NO_STATIC=1 && \ - make PREFIX=/usr NO_STATIC=1 install && \ - cd .. && rm -rf OpenBLAS-0.3.9 && rm v0.3.9.tar.gz - -ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" diff --git a/tests/README.md b/tests/README.md index 51b9fccbe6..7ac76f4db9 100644 --- a/tests/README.md +++ b/tests/README.md @@ -6,23 +6,23 @@ ##### Operating System -| Operating System | Version | -| ------ | --------- | -| CentOS | 7.5 or above | -| Ubuntu | 16.04 or above | -| Mac | 10.14 or above | +| Operating System | Version | +| ---------------- | -------------- | +| Amazon Linux | 2023 or above | +| Ubuntu | 20.04 or above | +| Mac | 10.14 or above | ##### Hardware -| Hardware Type | Recommended Configuration | -| ---- | --------------------------------------------------------------------------------------------------- | -| CPU | x86_64 architecture
Intel CPU Sandy Bridge or above
CPU Instruction Set
- SSE4_2
- AVX
- AVX2
- AVX512 | -| Memory | 16 GB or more | +| Hardware Type | Recommended Configuration | +| ------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------- | +| CPU | x86_64 architecture
Intel CPU Sandy Bridge or above
CPU Instruction Set
- SSE4_2
- AVX
- AVX2
- AVX512 or arm64 Linux/MacOS | +| Memory | 16 GB or more | ##### Software -| Software Name | Version | -| -------------- | ---------- | +| Software Name | Version | +| -------------- | --------------- | | Docker | 19.05 or above | | Docker Compose | 1.25.5 or above | | jq | 1.3 or above | @@ -34,19 +34,19 @@ ##### Troubleshooting Docker and Docker Compose - 1. Confirm that Docker Daemon is runningļ¼š +1. Confirm that Docker Daemon is runningļ¼š ```shell $ docker info ``` -- Ensure that Docker is installed. Refer to the official installation instructions for [Docker CE/EE](https://docs.docker.com/get-docker/). +- Ensure that Docker is installed. Refer to the official installation instructions for [Docker CE/EE](https://docs.docker.com/get-docker/). -- Start the Docker Daemon if it is not already started. +- Start the Docker Daemon if it is not already started. -- To run Docker without `root` privileges, create a user group labeled `docker`, then add a user to the group with `sudo usermod -aG docker $USER`. Log out and log back into the terminal for the changes to take effect. For more information, see the official Docker documentation for [Managing Docker as a Non-Root User](https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user). +- To run Docker without `root` privileges, create a user group labeled `docker`, then add a user to the group with `sudo usermod -aG docker $USER`. Log out and log back into the terminal for the changes to take effect. For more information, see the official Docker documentation for [Managing Docker as a Non-Root User](https://docs.docker.com/engine/install/linux-postinstall/#manage-docker-as-a-non-root-user). - 2. Check the version of Docker-Compose +2. Check the version of Docker-Compose ```shell $ docker-compose version @@ -57,23 +57,23 @@ CPython version: 3.7.5 OpenSSL version: OpenSSL 1.1.1f 31 Mar 2020 ``` -- To install Docker-Compose, see [Install Docker Compose](https://docs.docker.com/compose/install/) +- To install Docker-Compose, see [Install Docker Compose](https://docs.docker.com/compose/install/) ##### Install jq -- Refer to +- Refer to ##### Install kubectl -- Refer to +- Refer to ##### Install helm -- Refer to +- Refer to ##### Install kind -- Refer to +- Refer to #### Run E2E Tests