diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 280bec4486..9bfe15df5c 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -88,7 +88,7 @@ jobs: restore-keys: ubuntu${{ matrix.ubuntu }}-conan- - name: Build run: | - ./build/builder.sh /bin/bash -c "make USE_ASAN=${{env.useasan}} build-cpp-with-coverage generated-proto-go-without-cpp" + ./build/builder.sh /bin/bash -c "make USE_ASAN=${{env.useasan}} build-cpp-with-coverage" - run: | zip -r code.zip . -x "./.docker/*" - name: Archive code diff --git a/Makefile b/Makefile index b44c10f4b7..b048086066 100644 --- a/Makefile +++ b/Makefile @@ -21,9 +21,9 @@ OS := $(shell uname -s) ARCH := $(shell arch) mode = Release disk_index = OFF -useasan = false +useasan = false ifeq (${USE_ASAN}, true) -useasan = true +useasan = true endif export GIT_BRANCH=master @@ -58,19 +58,6 @@ tools/bin/revive: tools/check/go.mod cppcheck: @(env bash ${PWD}/scripts/core_build.sh -l) -# put generate proto as a separated target because build cpp have different cases like with unittest. -generated-proto-go-without-cpp: export protoc:=${PWD}/cmake_build/bin/protoc -generated-proto-go-without-cpp: - @mkdir -p ${GOPATH}/bin - @which protoc-gen-go 1>/dev/null || (echo "Installing protoc-gen-go" && cd /tmp && go install github.com/golang/protobuf/protoc-gen-go@v1.3.2) - @(env bash $(PWD)/scripts/proto_gen_go.sh) - -generated-proto-go: build-cpp generated-proto-go-without-cpp - -check-proto-product-only: - @(env bash $(PWD)/scripts/check_proto_product.sh) -check-proto-product: generated-proto-go check-proto-product-only - fmt: ifdef GO_DIFF_FILES @@ -150,13 +137,6 @@ print-gpu-build-info: @echo "Git Commit: $(GIT_COMMIT)" @echo "Go Version: $(GO_VERSION)" -embd-milvus: build-cpp-embd print-build-info - @echo "Building **Embedded** Milvus ..." - @source $(PWD)/scripts/setenv.sh && \ - mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \ - GO111MODULE=on $(GO) build -ldflags="-r /tmp/milvus/lib/ -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \ - ${AARCH64_FLAG} -buildmode=c-shared -o $(INSTALL_PATH)/embd-milvus.so $(PWD)/pkg/embedded/embedded.go 1>/dev/null - update-milvus-api: download-milvus-proto @echo "Update milvus/api version ..." @(env bash $(PWD)/scripts/update-api-version.sh) @@ -165,26 +145,35 @@ download-milvus-proto: @echo "Download milvus-proto repo ..." @(env bash $(PWD)/scripts/download_milvus_proto.sh) -build-cpp: download-milvus-proto +build-3rdparty: + @echo "Build 3rdparty ..." + @(env bash $(PWD)/scripts/3rdparty_build.sh) + +generated-proto: download-milvus-proto build-3rdparty + @echo "Generate proto ..." + @mkdir -p ${GOPATH}/bin + @which protoc-gen-go 1>/dev/null || (echo "Installing protoc-gen-go" && cd /tmp && go install github.com/golang/protobuf/protoc-gen-go@v1.3.2) + @(env bash $(PWD)/scripts/generate_proto.sh) + +build-cpp: generated-proto @echo "Building Milvus cpp library ..." @(env bash $(PWD)/scripts/core_build.sh -t ${mode} -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index}) -build-cpp-gpu: download-milvus-proto +build-cpp-gpu: generated-proto @echo "Building Milvus cpp gpu library ..." @(env bash $(PWD)/scripts/core_build.sh -t ${mode} -g -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index}) -build-cpp-embd: download-milvus-proto - @echo "Building **Embedded** Milvus cpp library ..." - @(env bash $(PWD)/scripts/core_build.sh -b -t ${mode} -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index}) - -build-cpp-with-unittest: download-milvus-proto +build-cpp-with-unittest: generated-proto @echo "Building Milvus cpp library with unittest ..." @(env bash $(PWD)/scripts/core_build.sh -t ${mode} -u -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index}) -build-cpp-with-coverage: download-milvus-proto +build-cpp-with-coverage: generated-proto @echo "Building Milvus cpp library with coverage and unittest ..." @(env bash $(PWD)/scripts/core_build.sh -t ${mode} -u -a ${useasan} -c -f "$(CUSTOM_THIRDPARTY_PATH)" -n ${disk_index}) +check-proto-product: generated-proto + @(env bash $(PWD)/scripts/check_proto_product.sh) + # Run the tests. unittest: test-cpp test-go @@ -274,11 +263,6 @@ codecov-cpp: build-cpp-with-coverage @echo "Running cpp coverage..." @(env bash $(PWD)/scripts/run_cpp_codecov.sh) -# Package docker image locally. -# TODO: fix error occur at starting up -docker: install - ./build/build_image.sh - # Build each component and install binary to $GOPATH/bin. install: milvus @echo "Installing binary to './bin'" @@ -298,7 +282,6 @@ gpu-install: milvus-gpu -cp -r -P $(PWD)/internal/core/output/lib64/*.so* $(LIBRARY_PATH) 2>/dev/null @echo "Installation successful." - clean: @echo "Cleaning up all the generated files" @find . -name '*.test' | xargs rm -fv @@ -351,10 +334,10 @@ generate-mockery: getdeps # internal/datanode $(PWD)/bin/mockery --name=Allocator --dir=$(PWD)/internal/datanode/allocator/ --output=$(PWD)/internal/datanode/allocator --filename=mock_allocator.go --with-expecter --structname=MockAllocator --outpkg=allocator --inpackage # internal/querycoordv2 - $(PWD)/bin/mockery --name=QueryNodeServer --dir=$(PWD)/internal/proto/querypb/ --output=$(PWD)/internal/querycoordv2/mocks --filename=mock_querynode.go --with-expecter --structname=MockQueryNodeServer - $(PWD)/bin/mockery --name=Broker --dir=$(PWD)/internal/querycoordv2/meta --output=$(PWD)/internal/querycoordv2/meta --filename=mock_broker.go --with-expecter --structname=MockBroker --outpkg=meta - $(PWD)/bin/mockery --name=Scheduler --dir=$(PWD)/internal/querycoordv2/task --output=$(PWD)/internal/querycoordv2/task --filename=mock_scheduler.go --with-expecter --structname=MockScheduler --outpkg=task --inpackage - $(PWD)/bin/mockery --name=Cluster --dir=$(PWD)/internal/querycoordv2/session --output=$(PWD)/internal/querycoordv2/session --filename=mock_cluster.go --with-expecter --structname=MockCluster --outpkg=session --inpackage + $(PWD)/bin/mockery --name=QueryNodeServer --dir=$(PWD)/internal/proto/querypb/ --output=$(PWD)/internal/querycoordv2/mocks --filename=mock_querynode.go --with-expecter --structname=MockQueryNodeServer + $(PWD)/bin/mockery --name=Broker --dir=$(PWD)/internal/querycoordv2/meta --output=$(PWD)/internal/querycoordv2/meta --filename=mock_broker.go --with-expecter --structname=MockBroker --outpkg=meta + $(PWD)/bin/mockery --name=Scheduler --dir=$(PWD)/internal/querycoordv2/task --output=$(PWD)/internal/querycoordv2/task --filename=mock_scheduler.go --with-expecter --structname=MockScheduler --outpkg=task --inpackage + $(PWD)/bin/mockery --name=Cluster --dir=$(PWD)/internal/querycoordv2/session --output=$(PWD)/internal/querycoordv2/session --filename=mock_cluster.go --with-expecter --structname=MockCluster --outpkg=session --inpackage $(PWD)/bin/mockery --name=Store --dir=$(PWD)/internal/querycoordv2/meta --output=$(PWD)/internal/querycoordv2/meta --filename=mock_store.go --with-expecter --structname=MockStore --outpkg=meta --inpackage $(PWD)/bin/mockery --name=Balance --dir=$(PWD)/internal/querycoordv2/balance --output=$(PWD)/internal/querycoordv2/balance --filename=mock_balancer.go --with-expecter --structname=MockBalancer --outpkg=balance --inpackage $(PWD)/bin/mockery --name=Controller --dir=$(PWD)/internal/querycoordv2/dist --output=$(PWD)/internal/querycoordv2/dist --filename=mock_controller.go --with-expecter --structname=MockController --outpkg=dist --inpackage @@ -370,7 +353,5 @@ generate-mockery: getdeps $(PWD)/bin/mockery --name=Manager --dir=$(PWD)/internal/querynodev2/cluster --output=$(PWD)/internal/querynodev2/cluster --filename=mock_manager.go --with-expecter --outpkg=cluster --structname=MockManager --inpackage $(PWD)/bin/mockery --name=Loader --dir=$(PWD)/internal/querynodev2/segments --output=$(PWD)/internal/querynodev2/segments --filename=mock_loader.go --with-expecter --outpkg=segments --structname=MockLoader --inpackage $(PWD)/bin/mockery --name=Worker --dir=$(PWD)/internal/querynodev2/cluster --output=$(PWD)/internal/querynodev2/cluster --filename=mock_worker.go --with-expecter --outpkg=worker --structname=MockWorker --inpackage - $(PWD)/bin/mockery --name=ShardDelegator --dir=$(PWD)/internal/querynodev2/delegator/ --output=$(PWD)/internal/querynodev2/delegator/ --filename=mock_delegator.go --with-expecter --outpkg=delegator --structname=MockShardDelegator --inpackage - -ci-ut: build-cpp-with-coverage generated-proto-go-without-cpp codecov-cpp codecov-go + $(PWD)/bin/mockery --name=ShardDelegator --dir=$(PWD)/internal/querynodev2/delegator/ --output=$(PWD)/internal/querynodev2/delegator/ --filename=mock_delegator.go --with-expecter --outpkg=delegator --structname=MockShardDelegator --inpackage diff --git a/internal/core/src/pb/CMakeLists.txt b/internal/core/src/pb/CMakeLists.txt index e819d4e9d4..3c00203cf4 100644 --- a/internal/core/src/pb/CMakeLists.txt +++ b/internal/core/src/pb/CMakeLists.txt @@ -10,51 +10,14 @@ # or implied. See the License for the specific language governing permissions and limitations under the License find_package(Protobuf REQUIRED) -set( proto_file_names - schema.proto - common.proto + +add_library(milvus_proto STATIC + common.pb.cc + index_cgo_msg.pb.cc + plan.pb.cc + schema.pb.cc + segcore.pb.cc ) - -set( PROTO_PATH "${MILVUS_SOURCE_DIR}/../../cmake_build/thirdparty/milvus-proto/proto/") - -FOREACH(proto_file ${proto_file_names}) - STRING( REGEX REPLACE "[^/]proto" "" proto_src_file_name ${proto_file} ) - LIST( APPEND milvus_proto_srcs "${proto_src_file_name}.pb.cc" ) - - ADD_CUSTOM_COMMAND( - OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/${proto_src_file_name}.pb.h" - "${CMAKE_CURRENT_SOURCE_DIR}/${proto_src_file_name}.pb.cc" - COMMAND protobuf::protoc --proto_path=${PROTO_PATH}/ - --cpp_out=${CMAKE_CURRENT_SOURCE_DIR}/ - ${proto_file} - DEPENDS ${PROTO_PATH}/${proto_file} ) -ENDFOREACH(proto_file) - -set( proto_file_names - segcore.proto - index_cgo_msg.proto - plan.proto - ) - -set( PROTO_PATH "${MILVUS_SOURCE_DIR}/../proto/") -set( PROTO_API_PATH "${MILVUS_SOURCE_DIR}/../../cmake_build/thirdparty/milvus-proto/proto/" ) - -FOREACH(proto_file ${proto_file_names}) - STRING( REGEX REPLACE "[^/]proto" "" proto_src_file_name ${proto_file} ) - LIST( APPEND milvus_proto_srcs "${proto_src_file_name}.pb.cc" ) - - ADD_CUSTOM_COMMAND( - OUTPUT "${CMAKE_CURRENT_SOURCE_DIR}/${proto_src_file_name}.pb.h" - "${CMAKE_CURRENT_SOURCE_DIR}/${proto_src_file_name}.pb.cc" - COMMAND protobuf::protoc --proto_path=${PROTO_PATH}/ - --proto_path=${PROTO_API_PATH}/ - --cpp_out=${CMAKE_CURRENT_SOURCE_DIR}/ - ${proto_file} - DEPENDS ${PROTO_PATH}/${proto_file} ) - -ENDFOREACH(proto_file) - -add_library(milvus_proto STATIC ${milvus_proto_srcs}) message(STATUS "milvus proto sources: " ${milvus_proto_srcs}) target_link_libraries( milvus_proto PUBLIC ${CONAN_LIBS} ) diff --git a/scripts/3rdparty_build.sh b/scripts/3rdparty_build.sh new file mode 100644 index 0000000000..a525445604 --- /dev/null +++ b/scripts/3rdparty_build.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash + +# Licensed to the LF AI & Data foundation under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" +CPP_SRC_DIR="${ROOT_DIR}/internal/core" +BUILD_OUTPUT_DIR="${ROOT_DIR}/cmake_build" + +if [[ ! -d ${BUILD_OUTPUT_DIR} ]]; then + mkdir ${BUILD_OUTPUT_DIR} +fi + +source ${ROOT_DIR}/scripts/setenv.sh +pushd ${BUILD_OUTPUT_DIR} + +export CONAN_REVISIONS_ENABLED=1 +if [[ ! `conan remote list` == *default-conan-local* ]]; then + conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local +fi +unameOut="$(uname -s)" +case "${unameOut}" in + Darwin*) + conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler=clang -s compiler.version=${llvm_version} -s compiler.libcxx=libc++ -s compiler.cppstd=17 || { echo 'conan install failed'; exit 1; } + ;; + Linux*) + echo "Running on ${OS_NAME}" + export CPU_TARGET=avx + GCC_VERSION=`gcc -dumpversion` + if [[ `gcc -v 2>&1 | sed -n 's/.*\(--with-default-libstdcxx-abi\)=\(\w*\).*/\2/p'` == "gcc4" ]]; then + conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler.version=${GCC_VERSION} || { echo 'conan install failed'; exit 1; } + else + conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler.version=${GCC_VERSION} -s compiler.libcxx=libstdc++11 || { echo 'conan install failed'; exit 1; } + fi + ;; + *) + echo "Cannot build on windows" + ;; +esac + +popd diff --git a/scripts/core_build.sh b/scripts/core_build.sh index 5f61af559d..3d078bb795 100755 --- a/scripts/core_build.sh +++ b/scripts/core_build.sh @@ -149,6 +149,7 @@ done if [[ ! -d ${BUILD_OUTPUT_DIR} ]]; then mkdir ${BUILD_OUTPUT_DIR} fi +source ${ROOT_DIR}/scripts/setenv.sh CMAKE_GENERATOR="Unix Makefiles" @@ -188,47 +189,6 @@ if [[ ${MAKE_CLEAN} == "ON" ]]; then exit 0 fi -export CONAN_REVISIONS_ENABLED=1 -conan remote add default-conan-local https://milvus01.jfrog.io/artifactory/api/conan/default-conan-local -unameOut="$(uname -s)" -case "${unameOut}" in - Darwin*) - # detect llvm version by valid list - for llvm_version in 15 14 NOT_FOUND ; do - if brew ls --versions llvm@${llvm_version} > /dev/null; then - break - fi - done - if [ "${llvm_version}" = "NOT_FOUND" ] ; then - echo "valid llvm(14 or 15) not installed" - exit 1 - fi - llvm_prefix="$(brew --prefix llvm@${llvm_version})" - export CLANG_TOOLS_PATH="${llvm_prefix}/bin" - export CC="ccache ${llvm_prefix}/bin/clang" - export CXX="ccache ${llvm_prefix}/bin/clang++" - export ASM="${llvm_prefix}/bin/clang" - export CFLAGS="-Wno-deprecated-declarations -I$(brew --prefix libomp)/include" - export CXXFLAGS=${CFLAGS} - export LDFLAGS="-L$(brew --prefix libomp)/lib" - - conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler=clang -s compiler.version=${llvm_version} -s compiler.libcxx=libc++ -s compiler.cppstd=17 || { echo 'conan install failed'; exit 1; } - ;; - Linux*) - echo "Running on ${OS_NAME}" - export CPU_TARGET=avx - GCC_VERSION=`gcc -dumpversion` - if [[ `gcc -v 2>&1 | sed -n 's/.*\(--with-default-libstdcxx-abi\)=\(\w*\).*/\2/p'` == "gcc4" ]]; then - conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler.version=${GCC_VERSION} || { echo 'conan install failed'; exit 1; } - else - conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler.version=${GCC_VERSION} -s compiler.libcxx=libstdc++11 || { echo 'conan install failed'; exit 1; } - fi - ;; - *) - echo "Cannot build on windows" - ;; -esac - arch=$(uname -m) CMAKE_CMD="cmake \ ${CMAKE_EXTRA_ARGS} \ @@ -250,6 +210,7 @@ ${CMAKE_EXTRA_ARGS} \ -DUSE_ASAN=${USE_ASAN} \ ${CPP_SRC_DIR}" +echo "CC $CC" echo ${CMAKE_CMD} ${CMAKE_CMD} -G "${CMAKE_GENERATOR}" diff --git a/scripts/generate_proto.sh b/scripts/generate_proto.sh new file mode 100755 index 0000000000..2551f586c9 --- /dev/null +++ b/scripts/generate_proto.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +# Licensed to the LF AI & Data foundation under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" + +PROTO_DIR=$ROOT_DIR/internal/proto/ +API_PROTO_DIR=$ROOT_DIR/cmake_build/thirdparty/milvus-proto/proto +CPP_SRC_DIR=$ROOT_DIR/internal/core +PROTOC_BIN=$ROOT_DIR/cmake_build/bin/protoc + +PROGRAM=$(basename "$0") +GOPATH=$(go env GOPATH) + +if [ -z $GOPATH ]; then + printf "Error: the environment variable GOPATH is not set, please set it before running %s\n" $PROGRAM > /dev/stderr + exit 1 +fi + +export PATH=${GOPATH}/bin:$PATH + +# official go code ship with the crate, so we need to generate it manually. +pushd ${PROTO_DIR} + +mkdir -p etcdpb +mkdir -p indexcgopb + +mkdir -p internalpb +mkdir -p rootcoordpb + +mkdir -p segcorepb +mkdir -p proxypb + +mkdir -p indexpb +mkdir -p datapb +mkdir -p querypb +mkdir -p planpb + +mkdir -p $ROOT_DIR/cmd/tools/migration/legacy/legacypb + +protoc_opt="${PROTOC_BIN} --proto_path=${API_PROTO_DIR} --proto_path=." + +${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./etcdpb etcd_meta.proto || { echo 'generate etcd_meta.proto failed'; exit 1; } +${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./indexcgopb index_cgo_msg.proto || { echo 'generate index_cgo_msg failed '; exit 1; } +${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./rootcoordpb root_coord.proto || { echo 'generate root_coord.proto failed'; exit 1; } +${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./internalpb internal.proto || { echo 'generate internal.proto failed'; exit 1; } +${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./proxypb proxy.proto|| { echo 'generate proxy.proto failed'; exit 1; } +${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./indexpb index_coord.proto|| { echo 'generate index_coord.proto failed'; exit 1; } +${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./datapb data_coord.proto|| { echo 'generate data_coord.proto failed'; exit 1; } +${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./querypb query_coord.proto|| { echo 'generate query_coord.proto failed'; exit 1; } +${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./planpb plan.proto|| { echo 'generate plan.proto failed'; exit 1; } +${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./segcorepb segcore.proto|| { echo 'generate segcore.proto failed'; exit 1; } + +${protoc_opt} --proto_path=$ROOT_DIR/cmd/tools/migration/legacy/ \ + --go_out=plugins=grpc,paths=source_relative:../../cmd/tools/migration/legacy/legacypb legacy.proto || { echo 'generate legacy.proto failed'; exit 1; } + +${protoc_opt} --cpp_out=$CPP_SRC_DIR/src/pb schema.proto|| { echo 'generate schema.proto failed'; exit 1; } +${protoc_opt} --cpp_out=$CPP_SRC_DIR/src/pb common.proto|| { echo 'generate common.proto failed'; exit 1; } +${protoc_opt} --cpp_out=$CPP_SRC_DIR/src/pb segcore.proto|| { echo 'generate segcore.proto failed'; exit 1; } +${protoc_opt} --cpp_out=$CPP_SRC_DIR/src/pb index_cgo_msg.proto|| { echo 'generate index_cgo_msg.proto failed'; exit 1; } +${protoc_opt} --cpp_out=$CPP_SRC_DIR/src/pb plan.proto|| { echo 'generate plan.proto failed'; exit 1; } + +popd + + diff --git a/scripts/proto_gen_go.sh b/scripts/proto_gen_go.sh deleted file mode 100755 index b8f2dfd173..0000000000 --- a/scripts/proto_gen_go.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env bash - -# Licensed to the LF AI & Data foundation under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -SCRIPTS_DIR=$(dirname "$0") - -PROTO_DIR=$SCRIPTS_DIR/../internal/proto/ -API_PROTO_DIR=$SCRIPTS_DIR/../cmake_build/thirdparty/milvus-proto/proto/ - -PROGRAM=$(basename "$0") -GOPATH=$(go env GOPATH) - -if [ -z $GOPATH ]; then - printf "Error: the environment variable GOPATH is not set, please set it before running %s\n" $PROGRAM > /dev/stderr - exit 1 -fi - -export PATH=${GOPATH}/bin:$PATH - -# official go code ship with the crate, so we need to generate it manually. -pushd ${PROTO_DIR} - -mkdir -p etcdpb -mkdir -p indexcgopb - -mkdir -p internalpb -mkdir -p rootcoordpb - -mkdir -p segcorepb -mkdir -p proxypb - -mkdir -p indexpb -mkdir -p datapb -mkdir -p querypb -mkdir -p planpb - -mkdir -p ../../cmd/tools/migration/legacy/legacypb - -protoc_opt="${protoc} --proto_path=${API_PROTO_DIR} --proto_path=/usr/local/include --proto_path=/usr/include --proto_path=." - -${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./etcdpb etcd_meta.proto -${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./indexcgopb index_cgo_msg.proto -${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./rootcoordpb root_coord.proto -${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./internalpb internal.proto -${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./proxypb proxy.proto -${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./indexpb index_coord.proto -${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./datapb data_coord.proto -${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./querypb query_coord.proto -${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./planpb plan.proto -${protoc_opt} --go_out=plugins=grpc,paths=source_relative:./segcorepb segcore.proto - -${protoc_opt} --proto_path=../../cmd/tools/migration/legacy/ \ - --go_out=plugins=grpc,paths=source_relative:../../cmd/tools/migration/legacy/legacypb legacy.proto - -popd diff --git a/scripts/setenv.sh b/scripts/setenv.sh index 98863a1c05..93c40085dd 100644 --- a/scripts/setenv.sh +++ b/scripts/setenv.sh @@ -17,11 +17,11 @@ # limitations under the License. # Exit immediately for non zero status -set -e +set +e SOURCE="${BASH_SOURCE[0]}" # fix on zsh environment -if [[ "$SOURCE" == "" ]]; then +if [[ "$SOURCE" == "" ]]; then SOURCE="$0" fi @@ -35,7 +35,7 @@ ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" unameOut="$(uname -s)" case "${unameOut}" in - Linux*) + Linux*) LIBJEMALLOC=$PWD/internal/core/output/lib/libjemalloc.so if test -f "$LIBJEMALLOC"; then export LD_PRELOAD="$LIBJEMALLOC" @@ -45,11 +45,30 @@ case "${unameOut}" in export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:$ROOT_DIR/internal/core/output/lib/pkgconfig:$ROOT_DIR/internal/core/output/lib64/pkgconfig" export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:$ROOT_DIR/internal/core/output/lib:$ROOT_DIR/internal/core/output/lib64" export RPATH=$LD_LIBRARY_PATH;; - Darwin*) + Darwin*) + # detect llvm version by valid list + for llvm_version in 15 14 NOT_FOUND ; do + if brew ls --versions llvm@${llvm_version} > /dev/null; then + break + fi + done + if [ "${llvm_version}" = "NOT_FOUND" ] ; then + echo "valid llvm(14 or 15) not installed" + exit 1 + fi + llvm_prefix="$(brew --prefix llvm@${llvm_version})" + export CLANG_TOOLS_PATH="${llvm_prefix}/bin" + export CC="ccache ${llvm_prefix}/bin/clang" + export CXX="ccache ${llvm_prefix}/bin/clang++" + export ASM="${llvm_prefix}/bin/clang" + export CFLAGS="-Wno-deprecated-declarations -I$(brew --prefix libomp)/include" + export CXXFLAGS=${CFLAGS} + export LDFLAGS="-L$(brew --prefix libomp)/lib" + export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:$ROOT_DIR/internal/core/output/lib/pkgconfig" export DYLD_LIBRARY_PATH=$ROOT_DIR/internal/core/output/lib export RPATH=$DYLD_LIBRARY_PATH;; - MINGW*) + MINGW*) extra_path=$(cygpath -w "$ROOT_DIR/internal/core/output/lib") export PKG_CONFIG_PATH="${PKG_CONFIG_PATH};${extra_path}\pkgconfig" export LD_LIBRARY_PATH=$extra_path