diff --git a/.github/mergify.yml b/.github/mergify.yml index a7009f4dda..c0bd2496f9 100644 --- a/.github/mergify.yml +++ b/.github/mergify.yml @@ -2,9 +2,9 @@ pull_request_rules: - name: Add needs-dco label when DCO check failed conditions: - or: - - base=master - - base=sql_beta - - base~=^2(\.\d+){2}$ + - base=master + - base=sql_beta + - base~=^2(\.\d+){2}$ - -status-success=DCO actions: label: @@ -19,9 +19,9 @@ pull_request_rules: - name: Add dco-passed label when DCO check passed conditions: - or: - - base=master - - base~=^2(\.\d+){2}$ - - base=sql_beta + - base=master + - base~=^2(\.\d+){2}$ + - base=sql_beta - status-success=DCO actions: label: @@ -33,15 +33,15 @@ pull_request_rules: - name: Test passed for code changed-master conditions: - or: - - base=sql_beta - - base=master - - "status-success=Code Checker AMD64 Ubuntu 20.04" - - "status-success=Build and test AMD64 Ubuntu 20.04" - - "status-success=Code Checker MacOS 12" - - "status-success=Code Checker CentOS 7" - - "status-success=cpu-e2e" - - "status-success=codecov/patch" - - "status-success=codecov/project" + - base=sql_beta + - base=master + - 'status-success=Build and test AMD64 Ubuntu 20.04' + - 'status-success=Code Checker AMD64 Ubuntu 20.04' + - 'status-success=Code Checker MacOS 12' + - 'status-success=Code Checker Amazonlinux 2023' + - 'status-success=cpu-e2e' + - 'status-success=codecov/patch' + - 'status-success=codecov/project' actions: label: add: @@ -49,13 +49,13 @@ pull_request_rules: - name: Test passed for code changed -2.*.* conditions: - base~=^2(\.\d+){2}$ - - "status-success=Code Checker AMD64 Ubuntu 20.04" - - "status-success=Build and test AMD64 Ubuntu 20.04" - - "status-success=Code Checker MacOS 12" - - "status-success=Code Checker CentOS 7" - - "status-success=cpu-e2e" - - "status-success=codecov/patch" - - "status-success=codecov/project" + - 'status-success=Code Checker AMD64 Ubuntu 20.04' + - 'status-success=Build and test AMD64 Ubuntu 20.04' + - 'status-success=Code Checker MacOS 12' + - 'status-success=Code Checker CentOS 7' + - 'status-success=cpu-e2e' + - 'status-success=codecov/patch' + - 'status-success=codecov/project' actions: label: add: @@ -63,37 +63,36 @@ pull_request_rules: - name: Test passed for tests changed conditions: - or: - - base=master - - base=sql_beta - - base~=^2(\.\d+){2}$ + - base=master + - base=sql_beta + - base~=^2(\.\d+){2}$ - -files~=^(?!tests\/python_client).+ - - "status-success=cpu-e2e" - actions: - label: - add: - - ci-passed - - - name: Test passed for docs changed only - conditions: - - or: - - base=master - - base=sql_beta - - base~=^2(\.\d+){2}$ - - -files~=^(?!.*\.(md)).*$ + - 'status-success=cpu-e2e' actions: label: add: - ci-passed + - name: Test passed for docs changed only + conditions: + - or: + - base=master + - base=sql_beta + - base~=^2(\.\d+){2}$ + - -files~=^(?!.*\.(md)).*$ + actions: + label: + add: + - ci-passed - name: Test passed for non go or c++ code changed conditions: - or: - - base=master - - base=sql_beta - - base~=^2(\.\d+){2}$ + - base=master + - base=sql_beta + - base~=^2(\.\d+){2}$ - -files~=^(?=.*((\.(go|h|cpp)|CMakeLists.txt|conanfile\.*))).*$ - - "status-success=cpu-e2e" + - 'status-success=cpu-e2e' actions: label: add: @@ -102,16 +101,16 @@ pull_request_rules: - name: Test passed for go unittest code changed-master conditions: - or: - - base=master - - base=sql_beta - - "status-success=Code Checker AMD64 Ubuntu 20.04" - - "status-success=Build and test AMD64 Ubuntu 20.04" - - "status-success=Code Checker MacOS 12" - - "status-success=Code Checker CentOS 7" - - "status-success=UT for Go (20.04)" + - base=master + - base=sql_beta + - 'status-success=Build and test AMD64 Ubuntu 20.04' + - 'status-success=Code Checker AMD64 Ubuntu 20.04' + - 'status-success=Code Checker MacOS 12' + - 'status-success=Code Checker Amazonlinux 2023' + - 'status-success=UT for Go (20.04)' - or: - - -files~=^(?!pkg\/..*test\.go).*$ - - -files~=^(?!internal\/.*_test\.go).*$ + - -files~=^(?!pkg\/..*test\.go).*$ + - -files~=^(?!internal\/.*_test\.go).*$ actions: label: add: @@ -119,10 +118,10 @@ pull_request_rules: - name: Test passed for go unittest code changed -2.*.* conditions: - base~=^2(\.\d+){2}$ - - "status-success=Code Checker AMD64 Ubuntu 20.04" - - "status-success=Build and test AMD64 Ubuntu 20.04" - - "status-success=Code Checker MacOS 12" - - "status-success=Code Checker CentOS 7" + - 'status-success=Code Checker AMD64 Ubuntu 20.04' + - 'status-success=Build and test AMD64 Ubuntu 20.04' + - 'status-success=Code Checker MacOS 12' + - 'status-success=Code Checker CentOS 7' - -files~=^(?!internal\/.*_test\.go).*$ actions: label: @@ -132,9 +131,9 @@ pull_request_rules: - name: Test passed for mergify changed conditions: - or: - - base=master - - base=sql_beta - - base~=^2(\.\d+){2}$ + - base=master + - base=sql_beta + - base~=^2(\.\d+){2}$ - -files~=^(?!\.github\/mergify\.yml).*$ actions: label: @@ -144,9 +143,9 @@ pull_request_rules: - name: Test passed for title skip e2e conditions: - or: - - base=master - - base=sql_beta - - base~=^2(\.\d+){2}$ + - base=master + - base=sql_beta + - base~=^2(\.\d+){2}$ - title~=\[skip e2e\] - label=kind/improvement - -files~=^(?=.*((\.(go|h|cpp)|CMakeLists.txt))).*$ @@ -155,16 +154,15 @@ pull_request_rules: add: - ci-passed - - name: Blocking PR if missing a related issue or doesn't have kind/improvement label conditions: - or: - - base=master - - base=sql_beta - - base~=^2(\.\d+){2}$ + - base=master + - base=sql_beta + - base~=^2(\.\d+){2}$ - and: - - -body~=\#[0-9]{1,6}(\s+|$) - - -body~=https://github.com/milvus-io/milvus/issues/[0-9]{1,6}(\s+|$) + - -body~=\#[0-9]{1,6}(\s+|$) + - -body~=https://github.com/milvus-io/milvus/issues/[0-9]{1,6}(\s+|$) - -label=kind/improvement - -title~=\[automated\] actions: @@ -175,24 +173,23 @@ pull_request_rules: message: | @{{author}} Please associate the related issue to the body of your Pull Request. (eg. “issue: #”) - - name: Dismiss block label if related issue be added into PR conditions: - or: - - and: - - or: - - base=master - - base=sql_beta - - base~=^2(\.\d+){2}$ - - or: - - body~=\#[0-9]{1,6}(\s+|$) - - body~=https://github.com/milvus-io/milvus/issues/[0-9]{1,6}(\s+|$) - - and: - - or: - - base=master - - base=sql_beta - - base~=^2(\.\d+){2}$ - - label=kind/improvement + - and: + - or: + - base=master + - base=sql_beta + - base~=^2(\.\d+){2}$ + - or: + - body~=\#[0-9]{1,6}(\s+|$) + - body~=https://github.com/milvus-io/milvus/issues/[0-9]{1,6}(\s+|$) + - and: + - or: + - base=master + - base=sql_beta + - base~=^2(\.\d+){2}$ + - label=kind/improvement actions: label: remove: @@ -203,8 +200,8 @@ pull_request_rules: - base=2.2.0 # - base~=^2(\.\d+){2}$ - and: - - -body~=pr\:\ \#[0-9]{1,6}(\s+|$) - - -body~=https://github.com/milvus-io/milvus/pull/[0-9]{1,6}(\s+|$) + - -body~=pr\:\ \#[0-9]{1,6}(\s+|$) + - -body~=https://github.com/milvus-io/milvus/pull/[0-9]{1,6}(\s+|$) - -label=kind/branch-feature - -title~=\[automated\] actions: @@ -215,28 +212,25 @@ pull_request_rules: message: | @{{author}} Please associate the related pr of master to the body of your Pull Request. (eg. “pr: #”) - - name: Dismiss block label if related pr be added into PR conditions: - base=2.2.0 # - base~=^2(\.\d+){2}$ - or: - - body~=pr\:\ \#[0-9]{1,6}(\s+|$) - - body~=https://github.com/milvus-io/milvus/pull/[0-9]{1,6}(\s+|$) - - label=kind/branch-feature + - body~=pr\:\ \#[0-9]{1,6}(\s+|$) + - body~=https://github.com/milvus-io/milvus/pull/[0-9]{1,6}(\s+|$) + - label=kind/branch-feature actions: label: remove: - do-not-merge/missing-related-pr - - - name: Dismiss block label if automated create PR conditions: - or: - - base=master - - base=sql_beta - - base~=^2(\.\d+){2}$ + - base=master + - base=sql_beta + - base~=^2(\.\d+){2}$ - title~=\[automated\] actions: label: @@ -247,13 +241,13 @@ pull_request_rules: - name: Test passed for skip e2e-master conditions: - or: - - base=master - - base=sql_beta + - base=master + - base=sql_beta - title~=\[skip e2e\] - - "status-success=Code Checker AMD64 Ubuntu 20.04" - - "status-success=Build and test AMD64 Ubuntu 20.04" - - "status-success=Code Checker MacOS 12" - - "status-success=Code Checker CentOS 7" + - 'status-success=Code Checker AMD64 Ubuntu 20.04' + - 'status-success=Build and test AMD64 Ubuntu 20.04' + - 'status-success=Code Checker MacOS 12' + - 'status-success=Code Checker Amazonlinux 2023' - files~=^(?=.*((\.(go|h|cpp)|CMakeLists.txt))).*$ actions: label: @@ -264,10 +258,10 @@ pull_request_rules: conditions: - base~=^2(\.\d+){2}$ - title~=\[skip e2e\] - - "status-success=Code Checker AMD64 Ubuntu 20.04" - - "status-success=Build and test AMD64 Ubuntu 20.04" - - "status-success=Code Checker MacOS 12" - - "status-success=Code Checker CentOS 7" + - 'status-success=Code Checker AMD64 Ubuntu 20.04' + - 'status-success=Build and test AMD64 Ubuntu 20.04' + - 'status-success=Code Checker MacOS 12' + - 'status-success=Code Checker CentOS 7' - files~=^(?=.*((\.(go|h|cpp)|CMakeLists.txt))).*$ actions: label: @@ -278,14 +272,14 @@ pull_request_rules: conditions: - label!=manual-pass - or: - - base=master - - base=sql_beta + - base=master + - base=sql_beta - files~=^(?=.*((\.(go|h|cpp)|CMakeLists.txt))).*$ - or: - - "status-success!=Code Checker AMD64 Ubuntu 20.04" - - "status-success!=Build and test AMD64 Ubuntu 20.04" - - "status-success!=Code Checker MacOS 12" - - "status-success!=Code Checker CentOS 7" + - 'status-success!=Code Checker AMD64 Ubuntu 20.04' + - 'status-success!=Build and test AMD64 Ubuntu 20.04' + - 'status-success!=Code Checker MacOS 12' + - 'status-success=Code Checker Amazonlinux 2023' actions: label: remove: @@ -296,10 +290,10 @@ pull_request_rules: - base~=^2(\.\d+){2}$ - files~=^(?=.*((\.(go|h|cpp)|CMakeLists.txt))).*$ - or: - - "status-success!=Code Checker AMD64 Ubuntu 20.04" - - "status-success!=Build and test AMD64 Ubuntu 20.04" - - "status-success!=Code Checker MacOS 12" - - "status-success!=Code Checker CentOS 7" + - 'status-success!=Code Checker AMD64 Ubuntu 20.04' + - 'status-success!=Build and test AMD64 Ubuntu 20.04' + - 'status-success!=Code Checker MacOS 12' + - 'status-success!=Code Checker CentOS 7' actions: label: remove: @@ -308,25 +302,24 @@ pull_request_rules: conditions: - label!=manual-pass - or: - - base=master - - base=sql_beta - - base~=^2(\.\d+){2}$ + - base=master + - base=sql_beta + - base~=^2(\.\d+){2}$ - -title~=\[skip e2e\] - files~=^(?!(internal\/.*_test\.go|.*\.md)).*$ - - "status-success!=cpu-e2e" + - 'status-success!=cpu-e2e' actions: label: remove: - ci-passed - - name: Add comment when jenkins job failed conditions: - or: - - base=master - - base=sql_beta - - base~=^2(\.\d+){2}$ - - "check-failure=cpu-e2e" + - base=master + - base=sql_beta + - base~=^2(\.\d+){2}$ + - 'check-failure=cpu-e2e' actions: comment: message: | @@ -335,11 +328,11 @@ pull_request_rules: - name: Add comment when code checker or ut failed -master conditions: - or: - - base=master - - base=sql_beta + - base=master + - base=sql_beta - or: - - "check-failure=Code Checker AMD64 Ubuntu 20.04" - - "check-failure=Build and test AMD64 Ubuntu 20.04" + - 'check-failure=Code Checker AMD64 Ubuntu 20.04' + - 'check-failure=Build and test AMD64 Ubuntu 20.04' actions: comment: message: | @@ -349,21 +342,20 @@ pull_request_rules: conditions: - base~=^2(\.\d+){2}$ - or: - - "check-failure=Code Checker AMD64 Ubuntu 20.04" - - "check-failure=Build and test AMD64 Ubuntu 20.04" + - 'check-failure=Code Checker AMD64 Ubuntu 20.04' + - 'check-failure=Build and test AMD64 Ubuntu 20.04' actions: comment: message: | @{{author}} ut workflow job failed, comment `rerun ut` can trigger the job again. - - name: Add `needs-rebase` label when more than one commit in pr conditions: - or: - - base=master - - base=sql_beta - - base~=^2(\.\d+){2}$ - - "#commits>1" + - base=master + - base=sql_beta + - base~=^2(\.\d+){2}$ + - '#commits>1' actions: comment: message: | @@ -375,10 +367,10 @@ pull_request_rules: - name: Remove `needs-rebase` label when only one commit in pr conditions: - or: - - base=master - - base=sql_beta - - base~=^2(\.\d+){2}$ - - "#commits=1" + - base=master + - base=sql_beta + - base~=^2(\.\d+){2}$ + - '#commits=1' actions: label: remove: diff --git a/.github/workflows/code-checker.yaml b/.github/workflows/code-checker.yaml index 329cc0d779..c801ad97bb 100644 --- a/.github/workflows/code-checker.yaml +++ b/.github/workflows/code-checker.yaml @@ -29,17 +29,11 @@ on: jobs: ubuntu: - name: ${{ matrix.name }} - runs-on: ${{ matrix.os }} + name: Code Checker AMD64 Ubuntu 20.04 + runs-on: ubuntu-latest timeout-minutes: 180 strategy: fail-fast: false - matrix: - include: - - name: Code Checker AMD64 Ubuntu 20.04 - os: ubuntu-20.04 - env: - UBUNTU: 20.04 steps: - name: Checkout uses: actions/checkout@v2 @@ -50,7 +44,6 @@ jobs: echo "corehash=${CORE_HASH}" >> $GITHUB_ENV echo "Set CCache hash to ${CORE_HASH}" - name: Cache CCache Volumes - # uses: actions/cache@v3 uses: pat-s/always-upload-cache@v3 with: path: .docker/amd64-ubuntu20.04-ccache @@ -63,21 +56,22 @@ jobs: key: ubuntu20.04-go-mod-${{ hashFiles('**/go.sum') }} restore-keys: ubuntu20.04-go-mod- - name: Cache Conan Packages - # uses: actions/cache@v3 uses: pat-s/always-upload-cache@v3 with: path: .docker/amd64-ubuntu20.04-conan key: ubuntu20.04-conan-${{ hashFiles('internal/core/conanfile.*') }} restore-keys: ubuntu20.04-conan- + # - name: Setup upterm session + # uses: lhotari/action-upterm@v1 - name: Code Check env: OS_NAME: 'ubuntu20.04' run: | ./build/builder.sh /bin/bash -c "make check-proto-product && make verifiers" - centos: - name: Code Checker CentOS 7 - # Run in centos docker - runs-on: ubuntu-20.04 + amazonlinux: + name: Code Checker Amazonlinux 2023 + # Run in amazonlinux docker + runs-on: ubuntu-latest timeout-minutes: 180 steps: - name: Checkout @@ -89,33 +83,31 @@ jobs: echo "corehash=${CORE_HASH}" >> $GITHUB_ENV echo "Set CCache hash to ${CORE_HASH}" - name: Cache CCache Volumes - # uses: actions/cache@v3 uses: pat-s/always-upload-cache@v3 with: - path: .docker/amd64-centos7-ccache - key: centos7-ccache-${{ env.corehash }} - restore-keys: centos7-ccache- + path: .docker/amd64-amazonlinux2023-ccache + key: amazonlinux2023-ccache-${{ env.corehash }} + restore-keys: amazonlinux2023-ccache- - name: Cache Third Party uses: actions/cache@v3 with: path: .docker/thirdparty - key: centos7-thirdparty-${{ hashFiles('internal/core/thirdparty/**') }} - restore-keys: centos7-thirdparty- + key: amazonlinux2023-thirdparty-${{ hashFiles('internal/core/thirdparty/**') }} + restore-keys: amazonlinux2023-thirdparty- - name: Cache Go Mod Volumes uses: actions/cache@v3 with: - path: .docker/amd64-centos7-go-mod - key: centos7-go-mod-${{ hashFiles('**/go.sum') }} - restore-keys: centos7-go-mod- + path: .docker/amd64-amazonlinux2023-go-mod + key: amazonlinux2023-go-mod-${{ hashFiles('**/go.sum') }} + restore-keys: amazonlinux2023-go-mod- - name: Cache Conan Packages - # uses: actions/cache@v3 uses: pat-s/always-upload-cache@v3 with: - path: .docker/amd64-centos7-conan - key: centos7-conan-${{ hashFiles('internal/core/conanfile.*') }} - # restore-keys: centos7-conan- + path: .docker/amd64-amazonlinux2023-conan + key: amazonlinux2023-conan-${{ hashFiles('internal/core/conanfile.*') }} + restore-keys: amazonlinux2023-conan- - name: Code Check env: - OS_NAME: 'centos7' + OS_NAME: 'amazonlinux2023' run: | ./build/builder.sh /bin/bash -c "make install" diff --git a/.github/workflows/publish-builder.yaml b/.github/workflows/publish-builder.yaml index e1c958b300..2f961e9c6f 100644 --- a/.github/workflows/publish-builder.yaml +++ b/.github/workflows/publish-builder.yaml @@ -18,15 +18,17 @@ on: jobs: publish-builder: - name: AMD64 ${{ matrix.os }} + name: ${{ matrix.arch }} ${{ matrix.os }} runs-on: ubuntu-latest timeout-minutes: 60 strategy: fail-fast: false matrix: - os: [ubuntu20.04, centos7] + os: [ubuntu20.04, amazonlinux2023] + arch: [amd64] env: OS_NAME: ${{ matrix.os }} + IMAGE_ARCH: ${{ matrix.arch }} steps: - name: Checkout uses: actions/checkout@v2 @@ -35,6 +37,8 @@ jobs: run: | echo "::set-output name=version::$(date +%Y%m%d)" echo "::set-output name=sha_short::$(git rev-parse --short HEAD)" + # - name: Setup upterm session + # uses: lhotari/action-upterm@v1 - name: Docker Pull shell: bash run: | diff --git a/.github/workflows/publish-milvus-base.yaml b/.github/workflows/publish-milvus-base.yaml deleted file mode 100644 index cda17c7ceb..0000000000 --- a/.github/workflows/publish-milvus-base.yaml +++ /dev/null @@ -1,81 +0,0 @@ -name: Publish Milvus Base Image -# TODO: do not trigger action for some document file update - -# This workflow is triggered on pushes or pull request to the repository. -on: - push: - # file paths to consider in the event. Optional; defaults to all. - paths: - - 'build/docker/milvus/ubuntu20.04/Dockerfile.base' - - '.github/workflows/publish-milvus-base.yaml' - pull_request: - # file paths to consider in the event. Optional; defaults to all. - paths: - - 'build/docker/milvus/ubuntu20.04/Dockerfile.base' - - '.github/workflows/publish-milvus-base.yaml' - -jobs: - publish-milvus-base: - name: milvus-base-ubuntu20.04 - runs-on: ubuntu-latest - env: - OS_NAME: ubuntu20.04 - timeout-minutes: 60 - steps: - - name: Checkout - uses: actions/checkout@v2 - - name: Generate docker image tag name - id: generate-tag-name - run: | - export tag=$(date +%Y%m%d)-$(git rev-parse --short HEAD) - echo $tag - echo "::set-output name=docker_tag::$tag" - - name: Docker Build - shell: bash - run: | - docker build -f "./build/docker/milvus/${OS_NAME}/Dockerfile.base" -t "milvusdb/milvus-base:${{ steps.generate-tag-name.outputs.docker_tag }}" . - - - name: Docker login - if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' - uses: azure/docker-login@v1 - with: - username: ${{ secrets.DOCKERHUB_USER }} - password: ${{ secrets.DOCKERHUB_TOKEN }} - - - name: Docker Push - if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' - continue-on-error: false - shell: bash - run: | - docker push milvusdb/milvus-base:${{ steps.generate-tag-name.outputs.docker_tag }} - - name: Update Milvus Base Image Changes - if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' - continue-on-error: true - shell: bash - run: | - sed -i "s#MILVUS_BASE_IMAGE_TAG=.*#MILVUS_BASE_IMAGE_TAG=\"${{ steps.generate-tag-name.outputs.docker_tag }}\"#g" build/docker/milvus/ubuntu20.04/Dockerfile - git config --local user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config --local user.name "github-actions[bot]" - git add build/docker/milvus/ubuntu20.04/Dockerfile - git commit -m "Update Milvus base image tag" - - name: Create Pull Request - id: cpr - if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' - continue-on-error: true - uses: peter-evans/create-pull-request@v3 - with: - token: ${{ secrets.ALL_CONTRIBUTORS_TOKEN }} - author: ${{ github.actor }} <${{ github.actor }}@users.noreply.github.com> - signoff: true - branch: update_base_${{ github.sha }} - delete-branch: true - title: '[automated] Update Milvus Base image Tag' - body: | - Update Update Milvus Base image Tag - See changes: https://github.com/milvus-io/milvus/commit/${{ github.sha }} - Signed-off-by: ${{ github.actor }} ${{ github.actor }}@users.noreply.github.com - - name: Check outputs - if: success() && github.event_name == 'push' && github.repository == 'milvus-io/milvus' - run: | - echo "Pull Request Number - ${{ steps.cpr.outputs.pull-request-number }}" - echo "Pull Request URL - ${{ steps.cpr.outputs.pull-request-url }}" diff --git a/build/build_image.sh b/build/build_image.sh index e6a4fd4740..ce8c62e695 100755 --- a/build/build_image.sh +++ b/build/build_image.sh @@ -28,20 +28,21 @@ OS_NAME="${OS_NAME:-ubuntu20.04}" MILVUS_IMAGE_REPO="${MILVUS_IMAGE_REPO:-milvusdb/milvus}" MILVUS_IMAGE_TAG="${MILVUS_IMAGE_TAG:-latest}" -MILVUS_BASE_IMAGE_REPO="${MILVUS_BASE_IMAGE_REPO:-milvusdb/milvus-base}" -MILVUS_BASE_IMAGE_TAG="local" +if [ -z "$IMAGE_ARCH" ]; then + MACHINE=$(uname -m) + if [ "$MACHINE" = "x86_64" ]; then + IMAGE_ARCH="amd64" + else + IMAGE_ARCH="arm64" + fi + +fi + BUILD_ARGS="" pushd "${toplevel}" -BUILD_BASE_IMAGE=${BUILD_BASE_IMAGE:-"false"} -# Seperate base dockerfile to ignore install dependencies when build milvus image -if [[ ${OS_NAME} == "ubuntu20.04" && ${BUILD_BASE_IMAGE} == "true" ]]; then - docker build -f "./build/docker/milvus/${OS_NAME}/Dockerfile.base" -t "${MILVUS_BASE_IMAGE_REPO}:${MILVUS_BASE_IMAGE_TAG}" . - BUILD_ARGS="--build-arg MILVUS_BASE_IMAGE_REPO=${MILVUS_BASE_IMAGE_REPO} --build-arg MILVUS_BASE_IMAGE_TAG=${MILVUS_BASE_IMAGE_TAG}" -fi - -docker build ${BUILD_ARGS} -f "./build/docker/milvus/${OS_NAME}/Dockerfile" -t "${MILVUS_IMAGE_REPO}:${MILVUS_IMAGE_TAG}" . +docker build ${BUILD_ARGS} --platform linux/${IMAGE_ARCH} -f "./build/docker/milvus/${OS_NAME}/Dockerfile" -t "${MILVUS_IMAGE_REPO}:${MILVUS_IMAGE_TAG}" . image_size=$(docker inspect ${MILVUS_IMAGE_REPO}:${MILVUS_IMAGE_TAG} -f '{{.Size}}'| awk '{ byte =$1 /1024/1024/1024; print byte " GB" }') diff --git a/build/builder.sh b/build/builder.sh index 235dfc73f9..f0086dfa19 100755 --- a/build/builder.sh +++ b/build/builder.sh @@ -5,6 +5,10 @@ set -euo pipefail # Absolute path to the toplevel milvus directory. toplevel=$(dirname "$(cd "$(dirname "${0}")"; pwd)") +if [[ -f "$toplevel/.env" ]]; then + export $(cat $toplevel/.env | xargs) +fi + export OS_NAME="${OS_NAME:-ubuntu20.04}" pushd "${toplevel}" @@ -19,34 +23,17 @@ if [[ "${1-}" == "down" ]]; then exit 0 fi -# Attempt to run in the container with the same UID/GID as we have on the host, -# as this results in the correct permissions on files created in the shared -# volumes. This isn't always possible, however, as IDs less than 100 are -# reserved by Debian, and IDs in the low 100s are dynamically assigned to -# various system users and groups. To be safe, if we see a UID/GID less than -# 500, promote it to 501. This is notably necessary on macOS Lion and later, -# where administrator accounts are created with a GID of 20. This solution is -# not foolproof, but it works well in practice. -uid=$(id -u) -gid=$(id -g) -[ "$uid" -lt 500 ] && uid=501 -[ "$gid" -lt 500 ] && gid=$uid - -mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/amd64-${OS_NAME}-ccache" -mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/amd64-${OS_NAME}-go-mod" -mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/amd64-${OS_NAME}-vscode-extensions" -mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/amd64-${OS_NAME}-conan" +mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-ccache" +mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-go-mod" +mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-vscode-extensions" +mkdir -p "${DOCKER_VOLUME_DIRECTORY:-.docker}/${IMAGE_ARCH}-${OS_NAME}-conan" chmod -R 777 "${DOCKER_VOLUME_DIRECTORY:-.docker}" docker-compose pull --ignore-pull-failures builder -if [[ "${CHECK_BUILDER:-}" == "1" ]]; then +if [[ "${CHECK_BUILDER:-1}" == "1" ]]; then docker-compose build builder fi -if [[ "$(id -u)" != "0" ]]; then - docker-compose run --no-deps --rm -u "$uid:$gid" builder "$@" -else - docker-compose run --no-deps --rm --entrypoint "/tini -- /entrypoint.sh" builder "$@" -fi +docker-compose run --no-deps --rm builder "$@" popd diff --git a/build/docker/builder/cpu/amazonlinux2023/Dockerfile b/build/docker/builder/cpu/amazonlinux2023/Dockerfile new file mode 100644 index 0000000000..b7b4f94c5f --- /dev/null +++ b/build/docker/builder/cpu/amazonlinux2023/Dockerfile @@ -0,0 +1,50 @@ +# Copyright (C) 2019-2022 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under the License. + +FROM amazonlinux:2023 + +ARG TARGETARCH + +RUN yum install -y wget g++ gcc gdb libstdc++-static git make unzip tar which \ + autoconf automake golang python3 python3-pip perl-FindBin texinfo \ + pkg-config libuuid-devel libaio && \ + rm -rf /var/cache/yum/* + +RUN pip3 install conan==1.58.0 + +RUN echo "target arch $TARGETARCH" +RUN if [ "$TARGETARCH" = "amd64" ]; then CMAKE_SUFFIX=x86_64; else CMAKE_SUFFIX=aarch64; fi &&\ + wget -qO- "https://cmake.org/files/v3.24/cmake-3.24.4-linux-$CMAKE_SUFFIX.tar.gz" | tar --strip-components=1 -xz -C /usr/local + +RUN mkdir /tmp/ccache && wget -qO- https://github.com/ccache/ccache/releases/download/v4.8.2/ccache-4.8.2.tar.gz | tar --strip-components=1 -xz -C /tmp/ccache &&\ + cd /tmp/ccache && mkdir build && cd build && cmake -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr .. && make && make install &&\ + rm -rf /tmp/ccache + +# https://github.com/golang/go/issues/22040 Due to this issue, ld.gold cannot be used in the ARM environment, so the official golang package cannot be used. You need to use the golang package that comes with yum. +# Install Go +# ENV GOPATH /go +# ENV GOROOT /usr/local/go +# ENV GO111MODULE on +# ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH +# RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.20.7.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \ +# mkdir -p "$GOPATH/src" "$GOPATH/bin" && \ +# go clean --modcache && \ +# chmod -R 777 "$GOPATH" && chmod -R a+w $(go env GOTOOLDIR) + +# refer: https://code.visualstudio.com/docs/remote/containers-advanced#_avoiding-extension-reinstalls-on-container-rebuild +RUN mkdir -p /home/milvus/.vscode-server/extensions \ + /home/milvus/.vscode-server-insiders/extensions \ + && chmod -R 777 /home/milvus + +COPY --chown=0:0 build/docker/builder/entrypoint.sh / + +ENTRYPOINT [ "/entrypoint.sh" ] +CMD ["tail", "-f", "/dev/null"] diff --git a/build/docker/builder/cpu/ubuntu20.04/Dockerfile b/build/docker/builder/cpu/ubuntu20.04/Dockerfile index d0bd698181..c32d6bf4f5 100644 --- a/build/docker/builder/cpu/ubuntu20.04/Dockerfile +++ b/build/docker/builder/cpu/ubuntu20.04/Dockerfile @@ -9,45 +9,33 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under the License. -FROM milvusdb/openblas:ubuntu20.04-20220914-179ea77 +FROM ubuntu:focal-20220426 -RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-certificates gnupg2 && \ - wget -qO- "https://cmake.org/files/v3.24/cmake-3.24.0-linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ - apt-get update && apt-get install -y --no-install-recommends \ - g++ gcc gfortran git make ccache libssl-dev zlib1g-dev unzip \ +ARG TARGETARCH + +RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-certificates gnupg2 \ + g++ gcc gdb gdbserver git make ccache libssl-dev zlib1g-dev unzip \ clang-format-10 clang-tidy-10 lcov libtool m4 autoconf automake python3 python3-pip \ - pkg-config uuid-dev libaio-dev libgoogle-perftools-dev && \ + pkg-config uuid-dev libaio-dev && \ apt-get remove --purge -y && \ rm -rf /var/lib/apt/lists/* RUN pip3 install conan==1.58.0 +RUN echo "target arch $TARGETARCH" +RUN if [ "$TARGETARCH" = "amd64" ]; then CMAKE_SUFFIX=x86_64; else CMAKE_SUFFIX=aarch64; fi &&\ + wget -qO- "https://cmake.org/files/v3.24/cmake-3.24.0-linux-$CMAKE_SUFFIX.tar.gz" | tar --strip-components=1 -xz -C /usr/local + # Install Go ENV GOPATH /go ENV GOROOT /usr/local/go ENV GO111MODULE on ENV PATH $GOPATH/bin:$GOROOT/bin:$PATH -RUN mkdir -p /usr/local/go && wget -qO- "https://golang.org/dl/go1.20.4.linux-amd64.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \ +RUN mkdir -p /usr/local/go && wget -qO- "https://go.dev/dl/go1.20.7.linux-$TARGETARCH.tar.gz" | tar --strip-components=1 -xz -C /usr/local/go && \ mkdir -p "$GOPATH/src" "$GOPATH/bin" && \ - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ${GOPATH}/bin v1.46.2 && \ - # export GO111MODULE=on && go get github.com/quasilyte/go-ruleguard/cmd/ruleguard@v0.2.1 && \ - go install github.com/ramya-rao-a/go-outline@latest && \ - go install golang.org/x/tools/gopls@latest && \ - go install github.com/uudashr/gopkgs/v2/cmd/gopkgs@latest && \ - go install github.com/go-delve/delve/cmd/dlv@latest && \ - go install honnef.co/go/tools/cmd/staticcheck@2022.1 && \ go clean --modcache && \ chmod -R 777 "$GOPATH" && chmod -R a+w $(go env GOTOOLDIR) -RUN ln -s /go/bin/dlv /go/bin/dlv-dap - -RUN apt-get update && apt-get install -y --no-install-recommends \ - gdb gdbserver && \ - apt-get remove --purge -y && \ - rm -rf /var/lib/apt/lists/* - -RUN echo 'root:root' | chpasswd - # refer: https://code.visualstudio.com/docs/remote/containers-advanced#_avoiding-extension-reinstalls-on-container-rebuild RUN mkdir -p /home/milvus/.vscode-server/extensions \ /home/milvus/.vscode-server-insiders/extensions \ @@ -55,10 +43,5 @@ RUN mkdir -p /home/milvus/.vscode-server/extensions \ COPY --chown=0:0 build/docker/builder/entrypoint.sh / -RUN wget -qO- "https://github.com/jeffoverflow/autouseradd/releases/download/1.2.0/autouseradd-1.2.0-amd64.tar.gz" | tar xz -C / --strip-components 1 - -RUN wget -O /tini https://github.com/krallin/tini/releases/download/v0.19.0/tini && \ - chmod +x /tini - -ENTRYPOINT [ "/tini", "--", "autouseradd", "--user", "milvus", "--", "/entrypoint.sh" ] +ENTRYPOINT [ "/entrypoint.sh" ] CMD ["tail", "-f", "/dev/null"] diff --git a/build/docker/milvus/amazonlinux2023/Dockerfile b/build/docker/milvus/amazonlinux2023/Dockerfile new file mode 100644 index 0000000000..4be0fb3a15 --- /dev/null +++ b/build/docker/milvus/amazonlinux2023/Dockerfile @@ -0,0 +1,34 @@ +# Copyright (C) 2019-2020 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under the License. + +FROM amazonlinux:2023 + +ARG TARGETARCH + +RUN yum install -y wget libgomp libaio libatomic && \ + rm -rf /var/cache/yum/* + +COPY ./bin/ /milvus/bin/ + +COPY ./configs/ /milvus/configs/ + +COPY ./lib/ /milvus/lib/ + +ENV PATH=/milvus/bin:$PATH +ENV LD_LIBRARY_PATH=/milvus/lib:$LD_LIBRARY_PATH:/usr/lib +ENV LD_PRELOAD=/milvus/lib/libjemalloc.so + +# Add Tini +ADD https://github.com/krallin/tini/releases/download/v0.19.0/tini-$TARGETARCH /tini +RUN chmod +x /tini +ENTRYPOINT ["/tini", "--"] + +WORKDIR /milvus diff --git a/build/docker/milvus/ubuntu20.04/Dockerfile b/build/docker/milvus/ubuntu20.04/Dockerfile index f5c2d63fd6..ae8650e4c2 100644 --- a/build/docker/milvus/ubuntu20.04/Dockerfile +++ b/build/docker/milvus/ubuntu20.04/Dockerfile @@ -8,9 +8,15 @@ # Unless required by applicable law or agreed to in writing, software distributed under the License # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under the License. -ARG MILVUS_BASE_IMAGE_REPO="milvusdb/milvus-base" -ARG MILVUS_BASE_IMAGE_TAG="20221114-b1a6660" -FROM ${MILVUS_BASE_IMAGE_REPO}:${MILVUS_BASE_IMAGE_TAG} + +FROM ubuntu:focal-20220426 + +ARG TARGETARCH + +RUN apt-get update && \ + apt-get install -y --no-install-recommends curl ca-certificates libaio-dev libgomp1 && \ + apt-get remove --purge -y && \ + rm -rf /var/lib/apt/lists/* COPY ./bin/ /milvus/bin/ @@ -21,3 +27,10 @@ COPY ./lib/ /milvus/lib/ ENV PATH=/milvus/bin:$PATH ENV LD_LIBRARY_PATH=/milvus/lib:$LD_LIBRARY_PATH:/usr/lib ENV LD_PRELOAD=/milvus/lib/libjemalloc.so + +# Add Tini +ADD https://github.com/krallin/tini/releases/download/v0.19.0/tini-$TARGETARCH /tini +RUN chmod +x /tini +ENTRYPOINT ["/tini", "--"] + +WORKDIR /milvus/ diff --git a/build/docker/milvus/ubuntu20.04/Dockerfile.base b/build/docker/milvus/ubuntu20.04/Dockerfile.base deleted file mode 100644 index 6b1d60abf0..0000000000 --- a/build/docker/milvus/ubuntu20.04/Dockerfile.base +++ /dev/null @@ -1,35 +0,0 @@ -# Copyright (C) 2019-2022 Zilliz. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under the License. - -FROM milvusdb/openblas:ubuntu20.04-20220914-179ea77 AS openblas - -#FROM alpine -FROM ubuntu:focal-20220426 - -ENV DEBIAN_FRONTEND=noninteractive -ENV TZ=UTC - -RUN apt-get update && \ - apt-get install -y --no-install-recommends curl libtbb-dev gfortran netcat iputils-ping ca-certificates liblapack3 libzstd-dev uuid-dev libaio-dev libboost-program-options-dev libboost-filesystem-dev && \ - apt-get remove --purge -y && \ - rm -rf /var/lib/apt/lists/* - -COPY --from=openblas /usr/local/lib/libopenblasp-r0.3.21.so /usr/lib/ - -RUN ln -s /usr/lib/libopenblasp-r0.3.21.so /usr/lib/libopenblas.so.3 && \ - ln -s /usr/lib/libopenblas.so.3 /usr/lib/libopenblas.so - -# Add Tini -ADD https://github.com/krallin/tini/releases/download/v0.19.0/tini /tini -RUN chmod +x /tini -ENTRYPOINT ["/tini", "--"] - -WORKDIR /milvus/ diff --git a/build/docker/openblas/ubuntu20.04/Dockerfile b/build/docker/openblas/ubuntu20.04/Dockerfile deleted file mode 100644 index 843f2ed627..0000000000 --- a/build/docker/openblas/ubuntu20.04/Dockerfile +++ /dev/null @@ -1,22 +0,0 @@ -FROM ubuntu:focal-20220426 - -# pipefail is enabled for proper error detection in the `wget | apt-key add` -# step -SHELL ["/bin/bash", "-o", "pipefail", "-c"] - -ENV DEBIAN_FRONTEND noninteractive - -RUN apt-get update && apt-get install -y --no-install-recommends wget ca-certificates gnupg2 && \ - apt-get update && apt-get install -y --no-install-recommends g++ gcc gfortran git make liblapack-dev && \ - apt-get remove --purge -y && \ - rm -rf /var/lib/apt/lists/* && \ - wget https://github.com/xianyi/OpenBLAS/archive/v0.3.21.tar.gz && \ - tar zxvf v0.3.21.tar.gz && cd OpenBLAS-0.3.21 && \ - make NO_STATIC=1 NO_LAPACK=1 NO_LAPACKE=1 NO_CBLAS=1 NO_AFFINITY=1 USE_OPENMP=1 \ - TARGET=HASWELL DYNAMIC_ARCH=1 \ - NUM_THREADS=64 MAJOR_VERSION=3 libs shared && \ - make PREFIX=/usr/local NUM_THREADS=64 MAJOR_VERSION=3 install && \ - rm -f /usr/local/include/cblas.h /usr/local/include/lapack* && \ - cd .. && rm -rf OpenBLAS-0.3.21 && rm v0.3.21.tar.gz - -ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" diff --git a/cmd/tools/config/generate.go b/cmd/tools/config/generate.go index 5873b1e2b2..a1a127d529 100644 --- a/cmd/tools/config/generate.go +++ b/cmd/tools/config/generate.go @@ -56,7 +56,7 @@ func collectRecursive(params *paramtable.ComponentParam, data *[]DocContent, val tag := val.Type().Field(j).Tag t := val.Type().Field(j).Type.String() if t == "paramtable.ParamItem" { - item := subVal.Interface().(paramtable.ParamItem) + item := subVal.Interface().(paramtable.ParamItem) //nolint:govet refreshable := tag.Get("refreshable") defaultValue := params.GetWithDefault(item.Key, item.DefaultValue) log.Debug("got key", zap.String("key", item.Key), zap.Any("value", defaultValue), zap.String("variable", val.Type().Field(j).Name)) diff --git a/docker-compose.yml b/docker-compose.yml index 522addbdf3..6166fe8c7b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,14 +9,14 @@ x-ccache: &ccache services: builder: - image: ${IMAGE_REPO}/milvus-env:${IMAGE_ARCH}-${OS_NAME}-${DATE_VERSION} + image: ${IMAGE_REPO}/milvus-env:${OS_NAME}-${DATE_VERSION} # Build devcontainer build: context: . dockerfile: build/docker/builder/cpu/${OS_NAME}/Dockerfile cache_from: - - ${IMAGE_REPO}/milvus-env:${IMAGE_ARCH}-${OS_NAME}-${LATEST_DATE_VERSION} - # user: {{ CURRENT_ID }} + - ${IMAGE_REPO}/milvus-env:${OS_NAME}-${LATEST_DATE_VERSION} + platform: linux/${IMAGE_ARCH} shm_size: 2G # expose 19530 port so we can directly access milvus inside build container # ports: @@ -45,14 +45,13 @@ services: make check-proto-product && make verifiers && make unittest" gpubuilder: - # image: harbor.milvus.io/milvus/milvus-env:gpu-amd64-ubuntu20.04-20230317-a1c7b0c - image: ${IMAGE_REPO}/milvus-env:gpu-${IMAGE_ARCH}-${OS_NAME}-${GPU_DATE_VERSION} + image: ${IMAGE_REPO}/milvus-env:gpu-${OS_NAME}-${GPU_DATE_VERSION} # Build devcontainer build: context: . dockerfile: build/docker/builder/gpu/${OS_NAME}/Dockerfile cache_from: - - ${IMAGE_REPO}/milvus-env:${IMAGE_ARCH}-${OS_NAME}-${LATEST_DATE_VERSION} + - ${IMAGE_REPO}/milvus-env:$${OS_NAME}-${LATEST_DATE_VERSION} # user: {{ CURRENT_ID }} shm_size: 2G # expose 19530 port so we can directly access milvus inside build container @@ -67,10 +66,10 @@ services: CONAN_USER_HOME: /home/milvus volumes: &builder-volumes-gpu - .:/go/src/github.com/milvus-io/milvus:delegated - - ${DOCKER_VOLUME_DIRECTORY:-.docker-gpu}/${IMAGE_ARCH}-${OS_NAME}-ccache:/ccache:delegated - - ${DOCKER_VOLUME_DIRECTORY:-.docker-gpu}/${IMAGE_ARCH}-${OS_NAME}-go-mod:/go/pkg/mod:delegated - - ${DOCKER_VOLUME_DIRECTORY:-.docker-gpu}/${IMAGE_ARCH}-${OS_NAME}-vscode-extensions:/home/milvus/.vscode-server/extensions:delegated - - ${DOCKER_VOLUME_DIRECTORY:-.docker-gpu}/${IMAGE_ARCH}-${OS_NAME}-conan:/home/milvus/.conan:delegated + - ${DOCKER_VOLUME_DIRECTORY:-.docker-gpu}/${OS_NAME}-ccache:/ccache:delegated + - ${DOCKER_VOLUME_DIRECTORY:-.docker-gpu}/${OS_NAME}-go-mod:/go/pkg/mod:delegated + - ${DOCKER_VOLUME_DIRECTORY:-.docker-gpu}/${OS_NAME}-vscode-extensions:/home/milvus/.vscode-server/extensions:delegated + - ${DOCKER_VOLUME_DIRECTORY:-.docker-gpu}/${OS_NAME}-conan:/home/milvus/.conan:delegated working_dir: '/go/src/github.com/milvus-io/milvus' depends_on: - etcd diff --git a/internal/core/cmake/ThirdPartyPackages.cmake b/internal/core/cmake/ThirdPartyPackages.cmake index 383e422a36..ff3ed132ce 100644 --- a/internal/core/cmake/ThirdPartyPackages.cmake +++ b/internal/core/cmake/ThirdPartyPackages.cmake @@ -19,65 +19,6 @@ foreach (DEPENDENCY ${MILVUS_THIRDPARTY_DEPENDENCIES}) endif () endforeach () -# ---------------------------------------------------------------------- -# Identify OS -if (UNIX) - if (APPLE) - set(CMAKE_OS_NAME "osx" CACHE STRING "Operating system name" FORCE) - else (APPLE) - ## Check for Debian GNU/Linux ________________ - find_file(DEBIAN_FOUND debian_version debconf.conf - PATHS /etc - ) - if (DEBIAN_FOUND) - set(CMAKE_OS_NAME "debian" CACHE STRING "Operating system name" FORCE) - endif (DEBIAN_FOUND) - ## Check for Fedora _________________________ - find_file(FEDORA_FOUND fedora-release - PATHS /etc - ) - if (FEDORA_FOUND) - set(CMAKE_OS_NAME "fedora" CACHE STRING "Operating system name" FORCE) - endif (FEDORA_FOUND) - ## Check for RedHat _________________________ - find_file(REDHAT_FOUND redhat-release inittab.RH - PATHS /etc - ) - if (REDHAT_FOUND) - set(CMAKE_OS_NAME "redhat" CACHE STRING "Operating system name" FORCE) - endif (REDHAT_FOUND) - ## Extra check for Ubuntu ____________________ - if (DEBIAN_FOUND) - ## At its core Ubuntu is a Debian system, with - ## a slightly altered configuration; hence from - ## a first superficial inspection a system will - ## be considered as Debian, which signifies an - ## extra check is required. - find_file(UBUNTU_EXTRA legal issue - PATHS /etc - ) - if (UBUNTU_EXTRA) - ## Scan contents of file - file(STRINGS ${UBUNTU_EXTRA} UBUNTU_FOUND - REGEX Ubuntu - ) - ## Check result of string search - if (UBUNTU_FOUND) - set(CMAKE_OS_NAME "ubuntu" CACHE STRING "Operating system name" FORCE) - set(DEBIAN_FOUND FALSE) - - find_program(LSB_RELEASE_EXEC lsb_release) - execute_process(COMMAND ${LSB_RELEASE_EXEC} -rs - OUTPUT_VARIABLE LSB_RELEASE_ID_SHORT - OUTPUT_STRIP_TRAILING_WHITESPACE - ) - STRING(REGEX REPLACE "\\." "_" UBUNTU_VERSION "${LSB_RELEASE_ID_SHORT}") - endif (UBUNTU_FOUND) - endif (UBUNTU_EXTRA) - endif (DEBIAN_FOUND) - endif (APPLE) -endif (UNIX) - # ---------------------------------------------------------------------- # thirdparty directory set(THIRDPARTY_DIR "${MILVUS_SOURCE_DIR}/thirdparty") diff --git a/internal/core/conanfile.py b/internal/core/conanfile.py index 0bb426587a..06979f91a6 100644 --- a/internal/core/conanfile.py +++ b/internal/core/conanfile.py @@ -1,4 +1,4 @@ -from conans import ConanFile, CMake +from conans import ConanFile class MilvusConan(ConanFile): @@ -57,6 +57,7 @@ class MilvusConan(ConanFile): default_options = { "librdkafka:shared": True, "librdkafka:zstd": True, + "librdkafka:sasl": False, "rocksdb:shared": True, "rocksdb:with_zstd": True, "arrow:parquet": True, @@ -72,8 +73,10 @@ class MilvusConan(ConanFile): "glog:shared": True, "prometheus-cpp:with_pull": False, "fmt:header_only": True, - "onnetbb:tbbmalloc": False, + "onetbb:tbbmalloc": False, "onetbb:tbbproxy": False, + "openblas:shared": True, + "openblas:dynamic_arch": True, } def configure(self): @@ -83,6 +86,13 @@ class MilvusConan(ConanFile): del self.options["folly"].use_sse4_2 self.options["arrow"].with_jemalloc = False + if self.settings.arch == "armv8": + self.options["openblas"].dynamic_arch = False + + def requirements(self): + if self.settings.os != "Macos": + # MacOS does not need openblas + self.requires("openblas/0.3.23@milvus/dev") def imports(self): self.copy("*.dylib", "../lib", "lib") diff --git a/internal/datanode/compactor_test.go b/internal/datanode/compactor_test.go index 29ee551253..a885debbe0 100644 --- a/internal/datanode/compactor_test.go +++ b/internal/datanode/compactor_test.go @@ -374,9 +374,9 @@ func TestCompactionTaskInnerMethods(t *testing.T) { mockbIO := &binlogIO{cm, alloc} paramtable.Get().Save(Params.CommonCfg.EntityExpirationTTL.Key, "0") - BinLogMaxSize := Params.DataNodeCfg.BinLogMaxSize + BinLogMaxSize := Params.DataNodeCfg.BinLogMaxSize.GetAsInt() defer func() { - Params.DataNodeCfg.BinLogMaxSize = BinLogMaxSize + paramtable.Get().Save(Params.DataNodeCfg.BinLogMaxSize.Key, fmt.Sprintf("%d", BinLogMaxSize)) }() paramtable.Get().Save(Params.DataNodeCfg.BinLogMaxSize.Key, "1") iData := genInsertDataWithExpiredTS() diff --git a/scripts/3rdparty_build.sh b/scripts/3rdparty_build.sh index a525445604..a99d9bc402 100644 --- a/scripts/3rdparty_build.sh +++ b/scripts/3rdparty_build.sh @@ -34,6 +34,8 @@ source ${ROOT_DIR}/scripts/setenv.sh pushd ${BUILD_OUTPUT_DIR} export CONAN_REVISIONS_ENABLED=1 +export CXXFLAGS=-Wno-error=address +export CFLAGS=-Wno-error=address if [[ ! `conan remote list` == *default-conan-local* ]]; then conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local fi diff --git a/scripts/gofmt.sh b/scripts/gofmt.sh index b486788968..69d40086e9 100755 --- a/scripts/gofmt.sh +++ b/scripts/gofmt.sh @@ -35,7 +35,7 @@ if [[ $# -ne 0 ]]; then files+=("$arg") fi if [ -d "$arg" ];then - for file in `find $arg -type f | grep "\.go$"`; do + for file in `find $arg -type f | grep "\.go$" | grep -v -e mock -e proto`; do files+=("$file") done fi