diff --git a/.github/mergify.yml b/.github/mergify.yml index 65fe04058a..28c64f66df 100644 --- a/.github/mergify.yml +++ b/.github/mergify.yml @@ -85,7 +85,7 @@ pull_request_rules: - or: - base=master - base~=^2(\.\d+){2}$ - - -files~=^(?=.*((\.(go|h|cpp)|CMakeLists.txt))).*$ + - -files~=^(?=.*((\.(go|h|cpp)|CMakeLists.txt|conanfile\.*))).*$ - "status-success=cpu-e2e" actions: label: @@ -320,4 +320,4 @@ pull_request_rules: actions: label: remove: - - needs-rebase \ No newline at end of file + - needs-rebase diff --git a/.github/workflows/code-checker.yaml b/.github/workflows/code-checker.yaml index 9dd293b8df..cdc0f054af 100644 --- a/.github/workflows/code-checker.yaml +++ b/.github/workflows/code-checker.yaml @@ -51,12 +51,6 @@ jobs: path: .docker/amd64-ubuntu20.04-ccache key: ubuntu20.04-ccache-${{ env.corehash }} restore-keys: ubuntu20.04-ccache- - - name: Cache Third Party - uses: actions/cache@v3 - with: - path: .docker/thirdparty - key: ubuntu20.04-thirdparty-${{ hashFiles('internal/core/thirdparty/**') }} - restore-keys: ubuntu20.04-thirdparty- - name: Cache Go Mod Volumes uses: actions/cache@v3 with: @@ -67,7 +61,7 @@ jobs: uses: actions/cache@v3 with: path: .docker/amd64-ubuntu20.04-conan - key: ubuntu20.04-conan-${{ hashFiles('internal/core/conanfile.txt') }} + key: ubuntu20.04-conan-${{ hashFiles('internal/core/conanfile.*') }} restore-keys: ubuntu20.04-conan- - name: Code Check env: diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index d752fd10bd..c748e6d86a 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -58,12 +58,6 @@ jobs: path: .docker/amd64-ubuntu${{ matrix.ubuntu }}-ccache key: ubuntu${{ matrix.ubuntu }}-ccache-${{ env.corehash }} restore-keys: ubuntu${{ matrix.ubuntu }}-ccache- - - name: Cache Third Party - uses: actions/cache@v3 - with: - path: .docker/thirdparty - key: ubuntu${{ matrix.ubuntu }}-thirdparty-${{ hashFiles('internal/core/thirdparty/**') }} - restore-keys: ubuntu${{ matrix.ubuntu }}-thirdparty- - name: Cache Go Mod Volumes uses: actions/cache@v3 with: @@ -74,7 +68,7 @@ jobs: uses: actions/cache@v3 with: path: .docker/amd64-ubuntu${{ matrix.ubuntu }}-conan - key: ubuntu${{ matrix.ubuntu }}-conan-${{ hashFiles('internal/core/conanfile.txt') }} + key: ubuntu${{ matrix.ubuntu }}-conan-${{ hashFiles('internal/core/conanfile.*') }} restore-keys: ubuntu${{ matrix.ubuntu }}-conan- - name: Start Service shell: bash @@ -114,7 +108,7 @@ jobs: - name: Display structure of code coverage results run: | ls -lah - + - name: Upload coverage to Codecov if: "github.repository == 'milvus-io/milvus'" uses: codecov/codecov-action@v3.1.1 diff --git a/internal/core/conanfile.py b/internal/core/conanfile.py new file mode 100644 index 0000000000..01e6738592 --- /dev/null +++ b/internal/core/conanfile.py @@ -0,0 +1,46 @@ +from conans import ConanFile, CMake + + +class MilvusConan(ConanFile): + + settings = "os", "compiler", "build_type", "arch" + requires = ( + "rocksdb/6.29.5", + "boost/1.80.0", + "onetbb/2021.7.0", + "zstd/1.5.2", + "arrow/8.0.1", + "openssl/1.1.1q", + "aws-sdk-cpp/1.9.234", + "benchmark/1.7.0", + "gtest/1.8.1", + "protobuf/3.9.1", + "rapidxml/1.13", + "yaml-cpp/0.7.0", + "marisa/0.2.6", + "zlib/1.2.13", + ) + generators = "cmake" + default_options = { + "rocksdb:shared": True, + "arrow:parquet": True, + "arrow:compute": True, + "arrow:with_zstd": True, + "aws-sdk-cpp:text-to-speech": False, + "aws-sdk-cpp:transfer": False, + "gtest:build_gmock": False, + "jemalloc:shared": True, + } + should_build = False + + def configure(self): + + # Macos M1 cannot use jemalloc + if self.settings.os != "Macos" or self.settings.arch in ("x86_64", "x86"): + self.requires("jemalloc/5.3.0") + + def imports(self): + self.copy("*.dylib", "../lib", "lib") + self.copy("*.dll", "../lib", "lib") + self.copy("*.so*", "../lib", "lib") + self.copy("*", "../bin", "bin") diff --git a/internal/core/conanfile.txt b/internal/core/conanfile.txt deleted file mode 100644 index 727f92bc81..0000000000 --- a/internal/core/conanfile.txt +++ /dev/null @@ -1,33 +0,0 @@ -[requires] -rocksdb/6.29.5 -boost/1.80.0 -onetbb/2021.3.0 -zstd/1.5.2 -arrow/8.0.1 -openssl/1.1.1q -aws-sdk-cpp/1.9.234 -benchmark/1.7.0 -gtest/1.8.1 -protobuf/3.9.1 -rapidxml/1.13 -yaml-cpp/0.7.0 -marisa/0.2.6 -zlib/1.2.13 - -[generators] -cmake - -[options] -rocksdb:shared=True -arrow:parquet=True -arrow:compute=True -arrow:with_zstd=True -aws-sdk-cpp:text-to-speech=False -aws-sdk-cpp:transfer=False -gtest:build_gmock=False - -[imports] -bin, * -> ../bin -lib, *.dylib -> ../lib -lib, *.dll -> ../lib -lib, *.so* -> ../lib diff --git a/scripts/setenv.sh b/scripts/setenv.sh index 577683dfb7..98863a1c05 100644 --- a/scripts/setenv.sh +++ b/scripts/setenv.sh @@ -34,9 +34,14 @@ ROOT_DIR="$( cd -P "$( dirname "$SOURCE" )/.." && pwd )" unameOut="$(uname -s)" - case "${unameOut}" in Linux*) + LIBJEMALLOC=$PWD/internal/core/output/lib/libjemalloc.so + if test -f "$LIBJEMALLOC"; then + export LD_PRELOAD="$LIBJEMALLOC" + else + echo "WARN: Cannot find $LIBJEMALLOC" + fi export PKG_CONFIG_PATH="${PKG_CONFIG_PATH}:$ROOT_DIR/internal/core/output/lib/pkgconfig:$ROOT_DIR/internal/core/output/lib64/pkgconfig" export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:$ROOT_DIR/internal/core/output/lib:$ROOT_DIR/internal/core/output/lib64" export RPATH=$LD_LIBRARY_PATH;;