From ea901393efca5161a94444489f70406e889ba01f Mon Sep 17 00:00:00 2001 From: Enwei Jiao Date: Tue, 28 Jun 2022 19:58:25 +0800 Subject: [PATCH] fix windows ldflags (#17807) Signed-off-by: Enwei Jiao --- Makefile | 21 +++---- .../core/thirdparty/rocksdb/CMakeLists.txt | 18 +++--- .../core/thirdparty/rocksdb/rocksdb-win.pc.in | 10 +++ scripts/install_deps_msys.sh | 20 +++++- scripts/run_go_codecov.sh | 14 +---- scripts/run_go_unittest.sh | 62 ++++++++----------- scripts/setenv.sh | 55 ++++++++++++++++ 7 files changed, 129 insertions(+), 71 deletions(-) create mode 100644 internal/core/thirdparty/rocksdb/rocksdb-win.pc.in create mode 100644 scripts/setenv.sh diff --git a/Makefile b/Makefile index 856de7decc..a1132b5eca 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,8 @@ GO ?= go PWD := $(shell pwd) -GOPATH := $(shell $(GO) env GOPATH) +GOPATH := $(shell $(GO) env GOPATH) +SHELL := /bin/bash OBJPREFIX := "github.com/milvus-io/milvus/cmd/milvus" INSTALL_PATH := $(PWD)/bin @@ -19,7 +20,6 @@ LIBRARY_PATH := $(PWD)/lib OS := $(shell uname -s) ARCH := $(shell arch) mode = Release -PKG_CONFIG = $(shell echo "${PKG_CONFIG_PATH}:`pwd`/internal/core/output/lib/pkgconfig:`pwd`/internal/core/output/lib64/pkgconfig") all: build-cpp build-go @@ -66,8 +66,8 @@ lint: tools/bin/revive static-check: @echo "Running $@ check" @GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean - @GO111MODULE=on PKG_CONFIG_PATH="$(PKG_CONFIG)" ${GOPATH}/bin/golangci-lint run --timeout=30m --config ./.golangci.yml ./internal/... - @GO111MODULE=on PKG_CONFIG_PATH="$(PKG_CONFIG)" ${GOPATH}/bin/golangci-lint run --timeout=30m --config ./.golangci.yml ./cmd/... + @source $(PWD)/scripts/setenv.sh && GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=30m --config ./.golangci.yml ./internal/... + @source $(PWD)/scripts/setenv.sh && GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=30m --config ./.golangci.yml ./cmd/... # @GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=30m --config ./.golangci.yml ./tests/go_client/... verifiers: build-cpp getdeps cppcheck fmt static-check @@ -96,18 +96,17 @@ print-build-info: milvus: build-cpp print-build-info @echo "Building Milvus ..." @echo "if build fails on Mac M1 machines, you probably need to rerun scripts/install_deps.sh and then run: \`export PKG_CONFIG_PATH=\"/opt/homebrew/opt/openssl@3/lib/pkgconfig\"\`" - @mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \ - export PKG_CONFIG_PATH="$(PKG_CONFIG)" && \ - export RPATH=$$(pkg-config --libs-only-L milvus_common | cut -c 3-) && \ - GO111MODULE=on $(GO) build -ldflags="-r $${RPATH} -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \ + @source $(PWD)/scripts/setenv.sh && \ + mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \ + GO111MODULE=on $(GO) build -ldflags="-r $${RPATH} -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \ ${APPLE_SILICON_FLAG} -o $(INSTALL_PATH)/milvus $(PWD)/cmd/main.go 1>/dev/null embd-milvus: build-cpp-embd print-build-info @echo "Building **Embedded** Milvus ..." @echo "if build fails on Mac M1 machines, rerun scripts/install_deps.sh and then run: \`export PKG_CONFIG_PATH=\"/opt/homebrew/opt/openssl@3/lib/pkgconfig\"\`" - @mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \ - export PKG_CONFIG_PATH="$(PKG_CONFIG)" && GO111MODULE=on $(GO) build \ - -ldflags="-r /tmp/milvus/lib/ -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \ + @source $(PWD)/scripts/setenv.sh && \ + mkdir -p $(INSTALL_PATH) && go env -w CGO_ENABLED="1" && \ + GO111MODULE=on $(GO) build -ldflags="-r /tmp/milvus/lib/ -X '$(OBJPREFIX).BuildTags=$(BUILD_TAGS)' -X '$(OBJPREFIX).BuildTime=$(BUILD_TIME)' -X '$(OBJPREFIX).GitCommit=$(GIT_COMMIT)' -X '$(OBJPREFIX).GoVersion=$(GO_VERSION)'" \ ${APPLE_SILICON_FLAG} -buildmode=c-shared -o $(INSTALL_PATH)/embd-milvus.so $(PWD)/pkg/embedded/embedded.go 1>/dev/null build-go: milvus diff --git a/internal/core/thirdparty/rocksdb/CMakeLists.txt b/internal/core/thirdparty/rocksdb/CMakeLists.txt index fef375de99..af8e2a6bdc 100644 --- a/internal/core/thirdparty/rocksdb/CMakeLists.txt +++ b/internal/core/thirdparty/rocksdb/CMakeLists.txt @@ -14,24 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -# if ( APPLE ) set( ROCKSDB_VERSION "6.27.3" ) set( ROCKSDB_SOURCE_URL "https://github.com/facebook/rocksdb/archive/v${ROCKSDB_VERSION}.tar.gz") -# else () -# set( ROCKSDB_VERSION "6.15.2" ) -# set( ROCKSDB_SOURCE_URL -# "https://github.com/facebook/rocksdb/archive/v${ROCKSDB_VERSION}.tar.gz") -# endif () macro( build_rocksdb ) message( STATUS "Building ROCKSDB-${ROCKSDB_VERSION} from source" ) - # if ( APPLE ) set ( ROCKSDB_MD5 "e4a0625f0cec82060e62c81b787a1124" ) - # else () - # set ( ROCKSDB_MD5 "67f9e04fda62af551dd039c37b1808ac" ) - # endif () if ( EMBEDDED_MILVUS ) message ( STATUS "Turning on fPIC while building embedded Milvus" ) @@ -80,4 +70,10 @@ macro( build_rocksdb ) endmacro() -build_rocksdb() +if (MSYS) + message(STATUS "Using rocksdb form system") + configure_file(rocksdb-win.pc.in "${CMAKE_CURRENT_BINARY_DIR}/rocksdb.pc" @ONLY) + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/rocksdb.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") +else (MSYS) + build_rocksdb() +endif (MSYS) diff --git a/internal/core/thirdparty/rocksdb/rocksdb-win.pc.in b/internal/core/thirdparty/rocksdb/rocksdb-win.pc.in new file mode 100644 index 0000000000..b0082aeea6 --- /dev/null +++ b/internal/core/thirdparty/rocksdb/rocksdb-win.pc.in @@ -0,0 +1,10 @@ +libdir=/mingw64/lib +includedir=/mingw64/include + +Name: Rocksdb +Description: Rocksdb +Version: @ROCKSDB_VERSION@ + +Libs: -L${libdir} -lrocksdb +Libs.private: -lz -lbz2 +Cflags: -I${includedir} diff --git a/scripts/install_deps_msys.sh b/scripts/install_deps_msys.sh index 9526c7ab81..37d6231892 100644 --- a/scripts/install_deps_msys.sh +++ b/scripts/install_deps_msys.sh @@ -7,7 +7,9 @@ if [[ "${MSYSTEM}" != "MINGW64" ]] ; then exit 1 fi -pacman -S --noconfirm --needed \ +pacmanInstall() +{ + pacman -S --noconfirm --needed \ git make tar dos2unix zip unzip patch \ mingw-w64-x86_64-toolchain \ mingw-w64-x86_64-make \ @@ -21,8 +23,20 @@ pacman -S --noconfirm --needed \ mingw-w64-x86_64-python2 \ mingw-w64-x86_64-diffutils \ mingw-w64-x86_64-arrow \ - mingw-w64-x86_64-go - # mingw-w64-x86_64-rocksdb + mingw-w64-x86_64-go \ + mingw-w64-x86_64-rocksdb +} + +updateKey() +{ + pacman-key --refresh-keys +} + +pacmanInstall || { + updateKey + pacmanInstall + +} # dummy empty dl, TODO: remove later diff --git a/scripts/run_go_codecov.sh b/scripts/run_go_codecov.sh index c3e72decbd..07fe007a08 100755 --- a/scripts/run_go_codecov.sh +++ b/scripts/run_go_codecov.sh @@ -22,16 +22,8 @@ FILE_COVERAGE_HTML="go_coverage.html" set -ex echo "mode: atomic" > ${FILE_COVERAGE_INFO} -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" - -export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:$ROOT_DIR/internal/core/output/lib/pkgconfig:$ROOT_DIR/internal/core/output/lib64/pkgconfig" -export RPATH=$(pkg-config --libs-only-L milvus_common | cut -c 3-) +BASEDIR=$(dirname "$0") +source $BASEDIR/setenv.sh # run unittest echo "Running unittest under ./internal" @@ -39,7 +31,7 @@ if [[ $(uname -s) == "Darwin" && "$(uname -m)" == "arm64" ]]; then APPLE_SILICON_FLAG="-tags dynamic" fi for d in $(go list ./internal/... | grep -v -e vendor -e kafka -e planparserv2/generated); do - go test -race ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d" + go test -race ${APPLE_SILICON_FLAG} -v -coverpkg=./... -coverprofile=profile.out -covermode=atomic "$d" if [ -f profile.out ]; then grep -v kafka profile.out | grep -v planparserv2/generated | sed '1d' >> ${FILE_COVERAGE_INFO} rm profile.out diff --git a/scripts/run_go_unittest.sh b/scripts/run_go_unittest.sh index 5b18223939..23d5bfdf24 100755 --- a/scripts/run_go_unittest.sh +++ b/scripts/run_go_unittest.sh @@ -19,16 +19,8 @@ # Exit immediately for non zero status set -e -SOURCE="${BASH_SOURCE[0]}" -while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink - DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" - SOURCE="$(readlink "$SOURCE")" - [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located -done -ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" - -export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:$ROOT_DIR/internal/core/output/lib/pkgconfig:$ROOT_DIR/internal/core/output/lib64/pkgconfig" -export RPATH=$(pkg-config --libs-only-L milvus_common | cut -c 3-) +BASEDIR=$(dirname "$0") +source $BASEDIR/setenv.sh if [[ $(uname -s) == "Darwin" ]]; then export MallocNanoZone=0 @@ -42,30 +34,30 @@ fi MILVUS_DIR="${ROOT_DIR}/internal/" echo "Running go unittest under $MILVUS_DIR" -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/allocator/..." -failfast -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/kv/..." -failfast -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" $(go list "${MILVUS_DIR}/mq/..." | grep -v kafka) -failfast -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/storage" -failfast -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/tso/..." -failfast -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/util/funcutil/..." -failfast -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/util/paramtable/..." -failfast -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/util/retry/..." -failfast -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/util/sessionutil/..." -failfast -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/util/trace/..." -failfast -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/util/typeutil/..." -failfast -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/util/importutil/..." -failfast -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/proxy/..." -failfast -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/datanode/..." -failfast -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/indexnode/..." -failfast -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/querynode/..." -failfast -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/distributed/rootcoord/..." -failfast -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/distributed/datacoord/..." -failfast -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/distributed/querycoord/..." -failfast -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/distributed/proxy/..." -failfast -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/distributed/datanode/..." -failfast -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/distributed/querynode/..." -failfast -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/rootcoord" -failfast -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/datacoord/..." -failfast -go test -race -cover ${APPLE_SILICON_FLAG} -ldflags="-r $RPATH" "${MILVUS_DIR}/indexcoord/..." -failfast +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/allocator/..." -failfast +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/kv/..." -failfast +go test -race -cover ${APPLE_SILICON_FLAG} $(go list "${MILVUS_DIR}/mq/..." | grep -v kafka) -failfast +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/storage" -failfast +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/tso/..." -failfast +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/funcutil/..." -failfast +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/paramtable/..." -failfast +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/retry/..." -failfast +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/sessionutil/..." -failfast +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/trace/..." -failfast +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/typeutil/..." -failfast +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/util/importutil/..." -failfast +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/proxy/..." -failfast +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/datanode/..." -failfast +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/indexnode/..." -failfast +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/querynode/..." -failfast +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/rootcoord/..." -failfast +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/datacoord/..." -failfast +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/querycoord/..." -failfast +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/proxy/..." -failfast +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/datanode/..." -failfast +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/distributed/querynode/..." -failfast +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/rootcoord" -failfast +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/datacoord/..." -failfast +go test -race -cover ${APPLE_SILICON_FLAG} "${MILVUS_DIR}/indexcoord/..." -failfast echo " Go unittest finished" diff --git a/scripts/setenv.sh b/scripts/setenv.sh new file mode 100644 index 0000000000..577683dfb7 --- /dev/null +++ b/scripts/setenv.sh @@ -0,0 +1,55 @@ +#!/bin/bash + +# Licensed to the LF AI & Data foundation under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Exit immediately for non zero status +set -e + +SOURCE="${BASH_SOURCE[0]}" +# fix on zsh environment +if [[ "$SOURCE" == "" ]]; then + SOURCE="$0" +fi + +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" + +unameOut="$(uname -s)" + + +case "${unameOut}" in + Linux*) + export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:$ROOT_DIR/internal/core/output/lib/pkgconfig:$ROOT_DIR/internal/core/output/lib64/pkgconfig" + export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:$ROOT_DIR/internal/core/output/lib:$ROOT_DIR/internal/core/output/lib64" + export RPATH=$LD_LIBRARY_PATH;; + Darwin*) + export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:$ROOT_DIR/internal/core/output/lib/pkgconfig" + export DYLD_LIBRARY_PATH=$ROOT_DIR/internal/core/output/lib + export RPATH=$DYLD_LIBRARY_PATH;; + MINGW*) + extra_path=$(cygpath -w "$ROOT_DIR/internal/core/output/lib") + export PKG_CONFIG_PATH="${PKG_CONFIG_PATH};${extra_path}\pkgconfig" + export LD_LIBRARY_PATH=$extra_path + export RPATH=$LD_LIBRARY_PATH;; + *) + echo "does not supported" +esac +