Optimize mac build time (#23390)

Signed-off-by: Enwei Jiao <enwei.jiao@zilliz.com>

.github/workflows/mac.yaml

@@ -37,23 +37,8 @@ jobs:
         run: |
           echo "corehash=${CORE_HASH}" >> $GITHUB_ENV
           echo "Set CCache hash to ${CORE_HASH}"
-      - name: Mac Cache Third Party
-        # uses: actions/cache@v3
-        uses: pat-s/always-upload-cache@v3
-        with:
-          path: |
-            /tmp/thirdparty
-          key: macos-thirdparty-${{ hashFiles('internal/core/thirdparty/**') }}
-          restore-keys: macos-thirdparty-
-      - name: Mac Cache Cmake Build
-        uses: actions/cache@v3
-        with:
-          path: |
-            cmake_build/3rdparty_download/download
-          key: macos-cmake-build-${{ env.corehash }}
-          restore-keys: macos-cmake-build-
       - name: Mac Cache CCache Volumes
-        uses: actions/cache@v3
+        uses: pat-s/always-upload-cache@v3
         with:
           path: /var/tmp/ccache
           key: macos-ccache-${{ env.corehash }}
@@ -81,11 +66,11 @@ jobs:
           CCACHE_COMPRESS: 1
           CCACHE_COMPRESSLEVEL: 5
           CCACHE_MAXSIZE: 2G
-          CUSTOM_THIRDPARTY_PATH: /tmp/thirdparty
         run: |
           if [[ ! -d "/var/tmp/ccache" ]];then
             mkdir -p /var/tmp/ccache
           fi
+          ls -alh /var/tmp/ccache
           brew install libomp ninja openblas ccache pkg-config
           pip3 install conan==1.58.0
           if [[ ! -d "/usr/local/opt/llvm" ]]; then

internal/core/conanfile.py

@@ -9,7 +9,7 @@ class MilvusConan(ConanFile):
         "boost/1.81.0",
         "onetbb/2021.7.0",
         "nlohmann_json/3.11.2",
-        "zstd/1.5.2",
+        "zstd/1.5.5",
         "lz4/1.9.4",
         "snappy/1.1.9",
         "lzo/2.10",
@@ -32,13 +32,10 @@ class MilvusConan(ConanFile):
         "libdwarf/20191104",
         "libiberty/9.1.0",
         "libsodium/cci.20220430",
-        "bison/3.5.3",
-        "flex/2.6.4",
         "xsimd/9.0.1",
         "xz_utils/5.4.0",
         "prometheus-cpp/1.1.0",
         "folly/2023.02.24@milvus/dev",
-        "velox/2023.02.24@milvus/dev",
         "opentelemetry-cpp/1.8.1.1@milvus/dev",
     )
     generators = ("cmake", "cmake_find_package")
@@ -54,6 +51,7 @@ class MilvusConan(ConanFile):
         "gtest:build_gmock": False,
         "boost:without_locale": False,
         "glog:with_gflags": False,
+        "prometheus-cpp:with_pull": False,
     }
 
     def configure(self):

internal/core/src/segcore/CMakeLists.txt

@@ -54,7 +54,3 @@ target_link_libraries(milvus_segcore
         )
 
 install(TARGETS milvus_segcore DESTINATION "${CMAKE_INSTALL_LIBDIR}")
-
-# add_executable(velox_demo VeloxDemo.cpp)
-# target_link_libraries(velox_demo ${CONAN_LIBS} velox_bundled)
-# install(TARGETS velox_demo DESTINATION "${CMAKE_INSTALL_BINDIR}")

internal/core/src/segcore/VeloxDemo.cpp

@@ -1,80 +0,0 @@
-/*
- * Copyright (c) Facebook, Inc. and its affiliates.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *     http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-#include <folly/init/Init.h>
-#include <algorithm>
-
-#include "velox/common/file/FileSystems.h"
-#include "velox/common/memory/Memory.h"
-#include "velox/dwio/dwrf/reader/DwrfReader.h"
-#include "velox/exec/tests/utils/TempDirectoryPath.h"
-#include "velox/vector/BaseVector.h"
-
-namespace filesystems = facebook::velox::filesystems;
-namespace dwio = facebook::velox::dwio;
-namespace dwrf = facebook::velox::dwrf;
-
-using facebook::velox::LocalReadFile;
-using facebook::velox::RowVector;
-using facebook::velox::vector_size_t;
-using facebook::velox::VectorPtr;
-
-using dwio::common::BufferedInput;
-using dwio::common::FileFormat;
-using dwio::common::ReaderOptions;
-using dwio::common::RowReaderOptions;
-
-using dwrf::DwrfReader;
-
-// A temporary program that reads from ORC file and prints its content
-// Used to compare the ORC data read by DWRFReader against apache-orc repo.
-// Usage: velox_example_scan_orc {orc_file_path}
-int
-main(int argc, char** argv) {
-    folly::init(&argc, &argv);
-
-    if (argc < 2) {
-        return 1;
-    }
-
-    // To be able to read local files, we need to register the local file
-    // filesystem. We also need to register the dwrf reader factory:
-    filesystems::registerLocalFileSystem();
-    dwrf::registerDwrfReaderFactory();
-    auto pool = facebook::velox::memory::getDefaultMemoryPool();
-
-    std::string filePath{argv[1]};
-    ReaderOptions readerOpts{pool.get()};
-    // To make DwrfReader reads ORC file, setFileFormat to FileFormat::ORC
-    readerOpts.setFileFormat(FileFormat::ORC);
-    auto reader =
-        DwrfReader::create(std::make_unique<BufferedInput>(
-                               std::make_shared<LocalReadFile>(filePath),
-                               readerOpts.getMemoryPool()),
-                           readerOpts);
-
-    VectorPtr batch;
-    RowReaderOptions rowReaderOptions;
-    auto rowReader = reader->createRowReader(rowReaderOptions);
-    while (rowReader->next(500, batch)) {
-        auto rowVector = batch->as<RowVector>();
-        for (vector_size_t i = 0; i < rowVector->size(); ++i) {
-            std::cout << rowVector->toString(i) << std::endl;
-        }
-    }
-
-    return 0;
-}

scripts/core_build.sh

@@ -205,8 +205,9 @@ case "${unameOut}" in
     fi
     llvm_prefix="$(brew --prefix llvm@${llvm_version})"
     export CLANG_TOOLS_PATH="${llvm_prefix}/bin"
-    export CC="${llvm_prefix}/bin/clang"
-    export CXX="${llvm_prefix}/bin/clang++"
+    export CC="ccache ${llvm_prefix}/bin/clang"
+    export CXX="ccache ${llvm_prefix}/bin/clang++"
+    export ASM="${llvm_prefix}/bin/clang"
     export CFLAGS="-Wno-deprecated-declarations -I$(brew --prefix libomp)/include"
     export CXXFLAGS=${CFLAGS}
     export LDFLAGS="-L$(brew --prefix libomp)/lib"
@@ -219,11 +220,11 @@ case "${unameOut}" in
     GCC_VERSION=`gcc -dumpversion`
     if [[ `gcc -v 2>&1 | sed -n 's/.*\(--with-default-libstdcxx-abi\)=\(\w*\).*/\2/p'` == "gcc4" ]]; then
       conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler.version=${GCC_VERSION} || { echo 'conan install failed'; exit 1; }
-    else 
+    else
       conan install ${CPP_SRC_DIR} --install-folder conan --build=missing -s compiler.version=${GCC_VERSION} -s compiler.libcxx=libstdc++11 || { echo 'conan install failed'; exit 1; }
-    fi 
+    fi
     ;;
-  *)   
+  *)
     echo "Cannot build on windows"
     ;;
 esac