Merge pull request #304 from ZhifengZhang-CN/0.6.0

Add Travis CI for Milvus

.travis.yml

@@ -0,0 +1,20 @@
+language: cpp
+sudo: required
+dist: bionic
+
+cache:
+  directories:
+    - $HOME/.ccache
+
+addons:
+  apt:
+    update: true
+
+before_install:
+  - source ci/travis/before-install.sh
+
+install:
+  - source $TRAVIS_BUILD_DIR/ci/travis/install_dependency.sh
+
+script:
+  - $TRAVIS_BUILD_DIR/ci/travis/travis_build.sh

ci/scripts/build.sh

@@ -0,0 +1,105 @@
+#!/bin/bash
+
+set -e
+
+SOURCE="${BASH_SOURCE[0]}"
+while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink
+  DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+  SOURCE="$(readlink "$SOURCE")"
+  [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located
+done
+SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )"
+
+MILVUS_CORE_DIR="${SCRIPTS_DIR}/../../core"
+CORE_BUILD_DIR="${MILVUS_CORE_DIR}/cmake_build"
+BUILD_TYPE="Debug"
+BUILD_UNITTEST="OFF"
+INSTALL_PREFIX="/opt/milvus"
+BUILD_COVERAGE="OFF"
+USE_JFROG_CACHE="OFF"
+RUN_CPPLINT="OFF"
+CPU_VERSION="ON"
+WITH_MKL="OFF"
+CUDA_COMPILER=/usr/local/cuda/bin/nvcc
+
+while getopts "o:t:b:gulcjmh" arg
+do
+        case $arg in
+             o)
+                INSTALL_PREFIX=$OPTARG
+                ;;
+             t)
+                BUILD_TYPE=$OPTARG # BUILD_TYPE
+                ;;
+             b)
+                CORE_BUILD_DIR=$OPTARG # CORE_BUILD_DIR
+                ;;
+             g)
+                CPU_VERSION="OFF";
+                ;;
+             u)
+                echo "Build and run unittest cases" ;
+                BUILD_UNITTEST="ON";
+                ;;
+             l)
+                RUN_CPPLINT="ON"
+                ;;
+             c)
+                BUILD_COVERAGE="ON"
+                ;;
+             j)
+                USE_JFROG_CACHE="ON"
+                ;;
+             m)
+                WITH_MKL="ON"
+                ;;
+             h) # help
+                echo "
+
+parameter:
+-o: install prefix(default: /opt/milvus)
+-t: build type(default: Debug)
+-b: core code build directory
+-g: gpu version
+-u: building unit test options(default: OFF)
+-l: run cpplint, clang-format and clang-tidy(default: OFF)
+-c: code coverage(default: OFF)
+-j: use jfrog cache build directory(default: OFF)
+-m: build with MKL(default: OFF)
+-h: help
+
+usage:
+./build.sh -o \${INSTALL_PREFIX} -t \${BUILD_TYPE} -b \${CORE_BUILD_DIR} [-u] [-l] [-c] [-j] [-m] [-h]
+                "
+                exit 0
+                ;;
+             ?)
+                echo "ERROR! unknown argument"
+        exit 1
+        ;;
+        esac
+done
+
+if [[ ! -d ${CORE_BUILD_DIR} ]]; then
+    mkdir ${CORE_BUILD_DIR}
+fi
+
+pushd ${CORE_BUILD_DIR}
+
+CMAKE_CMD="cmake \
+-DCMAKE_INSTALL_PREFIX=${INSTALL_PREFIX}
+-DCMAKE_BUILD_TYPE=${BUILD_TYPE} \
+-DCMAKE_CUDA_COMPILER=${CUDA_COMPILER} \
+-DMILVUS_CPU_VERSION=${CPU_VERSION} \
+-DBUILD_UNIT_TEST=${BUILD_UNITTEST} \
+-DBUILD_COVERAGE=${BUILD_COVERAGE} \
+-DUSE_JFROG_CACHE=${USE_JFROG_CACHE} \
+-DBUILD_FAISS_WITH_MKL=${WITH_MKL} \
+-DArrow_SOURCE=AUTO \
+${MILVUS_CORE_DIR}"
+echo ${CMAKE_CMD}
+${CMAKE_CMD}
+
+# compile and build
+make -j8 || exit 1
+make install || exit 1

ci/travis/before-install.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+set -ex
+
+if [[ "${TRAVIS_OS_NAME}" == "linux" ]]; then
+  export CCACHE_COMPRESS=1
+  export CCACHE_COMPRESSLEVEL=5
+  export CCACHE_COMPILERCHECK=content
+  export PATH=/usr/lib/ccache/:$PATH
+  ccache --show-stats
+fi
+
+set +ex

ci/travis/install_dependency.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+set -e
+
+wget -P /tmp https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
+
+sudo apt-key add /tmp/GPG-PUB-KEY-INTEL-SW-PRODUCTS-2019.PUB
+
+echo "deb https://apt.repos.intel.com/mkl all main" | \
+  sudo tee /etc/apt/sources.list.d/intel-mkl.list
+
+sudo wget -O /usr/share/keyrings/apache-arrow-keyring.gpg https://dl.bintray.com/apache/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-keyring.gpg
+
+sudo tee /etc/apt/sources.list.d/apache-arrow.list <<APT_LINE
+deb [arch=amd64 signed-by=/usr/share/keyrings/apache-arrow-keyring.gpg] https://dl.bintray.com/apache/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/ $(lsb_release --codename --short) main
+deb-src [signed-by=/usr/share/keyrings/apache-arrow-keyring.gpg] https://dl.bintray.com/apache/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/ $(lsb_release --codename --short) main
+APT_LINE
+
+sudo apt-get update -qq
+
+sudo apt-get install -y -q --no-install-recommends \
+  gfortran \
+  lsb-core \
+  libtool \
+  automake \
+  ccache \
+  pkg-config \
+  libarrow-dev \
+  libjemalloc-dev \
+  libboost-serialization-dev \
+  libboost-filesystem-dev \
+  libboost-system-dev \
+  libboost-regex-dev \
+  intel-mkl-gnu-2019.5-281 \
+  intel-mkl-core-2019.5-281 \
+  libmysqlclient-dev \
+  clang-format-6.0 \
+  clang-tidy-6.0 \
+  lcov
+
+sudo ln -s /usr/lib/x86_64-linux-gnu/libmysqlclient.so \
+  /usr/lib/x86_64-linux-gnu/libmysqlclient_r.so
+
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/opt/intel/compilers_and_libraries_2019.5.281/linux/mkl/lib/intel64

ci/travis/travis_build.sh

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+set -ex
+
+source $TRAVIS_BUILD_DIR/ci/travis/travis_env_common.sh
+
+only_library_mode=no
+
+while true; do
+    case "$1" in
+	--only-library)
+	    only_library_mode=yes
+	    shift ;;
+	*) break ;;
+    esac
+done
+
+BUILD_COMMON_FLAGS="-t ${MILVUS_BUILD_TYPE} -o ${MILVUS_INSTALL_PREFIX} -b ${MILVUS_BUILD_DIR}"
+
+if [ $only_library_mode == "yes" ]; then
+  ${TRAVIS_BUILD_DIR}/ci/scripts/build.sh ${BUILD_COMMON_FLAGS} -m
+else
+  ${TRAVIS_BUILD_DIR}/ci/scripts/build.sh ${BUILD_COMMON_FLAGS} -m -u -c
+fi

ci/travis/travis_env_common.sh

@@ -0,0 +1,10 @@
+export MILVUS_CORE_DIR=${TRAVIS_BUILD_DIR}/core
+export MILVUS_BUILD_DIR=${TRAVIS_BUILD_DIR}/core/cmake_build
+export MILVUS_INSTALL_PREFIX=/opt/milvus
+export MILVUS_TRAVIS_COVERAGE=${MILVUS_TRAVIS_COVERAGE:=0}
+
+if [ "${MILVUS_TRAVIS_COVERAGE}" == "1" ]; then
+  export MILVUS_CPP_COVERAGE_FILE=${TRAVIS_BUILD_DIR}/output_new.info
+fi
+
+export MILVUS_BUILD_TYPE=${MILVUS_BUILD_TYPE:=Release}

core/CMakeLists.txt

@@ -18,7 +18,7 @@
 #-------------------------------------------------------------------------------
 
 
-cmake_minimum_required(VERSION 3.14)
+cmake_minimum_required(VERSION 3.12)
 message(STATUS "Building using CMake version: ${CMAKE_VERSION}")
 
 set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
@@ -118,6 +118,18 @@ include(DefineOptions)
 include(BuildUtils)
 include(ThirdPartyPackages)
 
+if(MILVUS_USE_CCACHE)
+  find_program(CCACHE_FOUND ccache)
+  if(CCACHE_FOUND)
+    message(STATUS "Using ccache: ${CCACHE_FOUND}")
+    set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE ${CCACHE_FOUND})
+    set_property(GLOBAL PROPERTY RULE_LAUNCH_LINK ${CCACHE_FOUND})
+    # let ccache preserve C++ comments, because some of them may be
+    # meaningful to the compiler
+    set(ENV{CCACHE_COMMENTS} "1")
+  endif(CCACHE_FOUND)
+endif()
+
 set(MILVUS_CPU_VERSION false)
 if (MILVUS_GPU_VERSION)
     message(STATUS "Building Milvus GPU version")
@@ -273,4 +285,4 @@ if (${CLANG_TIDY_FOUND})
             --source_dir
             ${CMAKE_CURRENT_SOURCE_DIR}/src
             ${MILVUS_LINT_QUIET})
-endif ()
+endif ()

core/cmake/DefineOptions.cmake

@@ -48,7 +48,7 @@ define_option(MILVUS_GPU_VERSION "Build GPU version" OFF)
 #----------------------------------------------------------------------
 set_option_category("Thirdparty")
 
-set(MILVUS_DEPENDENCY_SOURCE_DEFAULT "AUTO")
+set(MILVUS_DEPENDENCY_SOURCE_DEFAULT "BUNDLED")
 
 define_option_string(MILVUS_DEPENDENCY_SOURCE
         "Method to use for acquiring MILVUS's build dependencies"
@@ -57,6 +57,8 @@ define_option_string(MILVUS_DEPENDENCY_SOURCE
         "BUNDLED"
         "SYSTEM")
 
+define_option(MILVUS_USE_CCACHE "Use ccache when compiling (if available)" ON)
+
 define_option(MILVUS_VERBOSE_THIRDPARTY_BUILD
         "Show output from ExternalProjects rather than just logging to files" ON)
 

core/cmake/ThirdPartyPackages.cmake

@@ -164,8 +164,10 @@ endif ()
 
 macro(resolve_dependency DEPENDENCY_NAME)
     if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO")
-        #disable find_package for now
-        build_dependency(${DEPENDENCY_NAME})
+        find_package(${DEPENDENCY_NAME} MODULE)
+        if(NOT ${${DEPENDENCY_NAME}_FOUND})
+          build_dependency(${DEPENDENCY_NAME})
+        endif()
     elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED")
         build_dependency(${DEPENDENCY_NAME})
     elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM")

core/src/index/CMakeLists.txt

@@ -18,7 +18,7 @@
 #-------------------------------------------------------------------------------
 
 
-cmake_minimum_required(VERSION 3.14)
+cmake_minimum_required(VERSION 3.12)
 message(STATUS "------------------------------KNOWHERE-----------------------------------")
 message(STATUS "Building using CMake version: ${CMAKE_VERSION}")
 

core/src/index/cmake/DefineOptionsCore.cmake

@@ -52,7 +52,7 @@ endif ()
 #----------------------------------------------------------------------
 set_option_category("Thirdparty")
 
-set(KNOWHERE_DEPENDENCY_SOURCE_DEFAULT "AUTO")
+set(KNOWHERE_DEPENDENCY_SOURCE_DEFAULT "BUNDLED")
 
 define_option_string(KNOWHERE_DEPENDENCY_SOURCE
         "Method to use for acquiring KNOWHERE's build dependencies"

core/src/index/cmake/FindArrow.cmake

@@ -0,0 +1,431 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# - Find Arrow (arrow/api.h, libarrow.a, libarrow.so)
+# This module defines
+#  ARROW_FOUND, whether Arrow has been found
+#  ARROW_FULL_SO_VERSION, full shared object version of found Arrow "100.0.0"
+#  ARROW_IMPORT_LIB, path to libarrow's import library (Windows only)
+#  ARROW_INCLUDE_DIR, directory containing headers
+#  ARROW_LIBS, deprecated. Use ARROW_LIB_DIR instead
+#  ARROW_LIB_DIR, directory containing Arrow libraries
+#  ARROW_SHARED_IMP_LIB, deprecated. Use ARROW_IMPORT_LIB instead
+#  ARROW_SHARED_LIB, path to libarrow's shared library
+#  ARROW_SO_VERSION, shared object version of found Arrow such as "100"
+#  ARROW_STATIC_LIB, path to libarrow.a
+#  ARROW_VERSION, version of found Arrow
+#  ARROW_VERSION_MAJOR, major version of found Arrow
+#  ARROW_VERSION_MINOR, minor version of found Arrow
+#  ARROW_VERSION_PATCH, patch version of found Arrow
+
+include(FindPkgConfig)
+include(FindPackageHandleStandardArgs)
+
+set(ARROW_SEARCH_LIB_PATH_SUFFIXES)
+if(CMAKE_LIBRARY_ARCHITECTURE)
+  list(APPEND ARROW_SEARCH_LIB_PATH_SUFFIXES "lib/${CMAKE_LIBRARY_ARCHITECTURE}")
+endif()
+list(APPEND ARROW_SEARCH_LIB_PATH_SUFFIXES
+            "lib64"
+            "lib32"
+            "lib"
+            "bin")
+set(ARROW_CONFIG_SUFFIXES
+    "_RELEASE"
+    "_RELWITHDEBINFO"
+    "_MINSIZEREL"
+    "_DEBUG"
+    "")
+if(CMAKE_BUILD_TYPE)
+  string(TOUPPER ${CMAKE_BUILD_TYPE} ARROW_CONFIG_SUFFIX_PREFERRED)
+  set(ARROW_CONFIG_SUFFIX_PREFERRED "_${ARROW_CONFIG_SUFFIX_PREFERRED}")
+  list(INSERT ARROW_CONFIG_SUFFIXES 0 "${ARROW_CONFIG_SUFFIX_PREFERRED}")
+endif()
+
+if(NOT DEFINED ARROW_MSVC_STATIC_LIB_SUFFIX)
+  if(MSVC)
+    set(ARROW_MSVC_STATIC_LIB_SUFFIX "_static")
+  else()
+    set(ARROW_MSVC_STATIC_LIB_SUFFIX "")
+  endif()
+endif()
+
+# Internal function.
+#
+# Set shared library name for ${base_name} to ${output_variable}.
+#
+# Example:
+#   arrow_build_shared_library_name(ARROW_SHARED_LIBRARY_NAME arrow)
+#   # -> ARROW_SHARED_LIBRARY_NAME=libarrow.so on Linux
+#   # -> ARROW_SHARED_LIBRARY_NAME=libarrow.dylib on macOS
+#   # -> ARROW_SHARED_LIBRARY_NAME=arrow.dll with MSVC on Windows
+#   # -> ARROW_SHARED_LIBRARY_NAME=libarrow.dll with MinGW on Windows
+function(arrow_build_shared_library_name output_variable base_name)
+  set(${output_variable}
+      "${CMAKE_SHARED_LIBRARY_PREFIX}${base_name}${CMAKE_SHARED_LIBRARY_SUFFIX}"
+      PARENT_SCOPE)
+endfunction()
+
+# Internal function.
+#
+# Set import library name for ${base_name} to ${output_variable}.
+# This is useful only for MSVC build. Import library is used only
+# with MSVC build.
+#
+# Example:
+#   arrow_build_import_library_name(ARROW_IMPORT_LIBRARY_NAME arrow)
+#   # -> ARROW_IMPORT_LIBRARY_NAME=arrow on Linux (meaningless)
+#   # -> ARROW_IMPORT_LIBRARY_NAME=arrow on macOS (meaningless)
+#   # -> ARROW_IMPORT_LIBRARY_NAME=arrow.lib with MSVC on Windows
+#   # -> ARROW_IMPORT_LIBRARY_NAME=libarrow.dll.a with MinGW on Windows
+function(arrow_build_import_library_name output_variable base_name)
+  set(${output_variable}
+      "${CMAKE_IMPORT_LIBRARY_PREFIX}${base_name}${CMAKE_IMPORT_LIBRARY_SUFFIX}"
+      PARENT_SCOPE)
+endfunction()
+
+# Internal function.
+#
+# Set static library name for ${base_name} to ${output_variable}.
+#
+# Example:
+#   arrow_build_static_library_name(ARROW_STATIC_LIBRARY_NAME arrow)
+#   # -> ARROW_STATIC_LIBRARY_NAME=libarrow.a on Linux
+#   # -> ARROW_STATIC_LIBRARY_NAME=libarrow.a on macOS
+#   # -> ARROW_STATIC_LIBRARY_NAME=arrow.lib with MSVC on Windows
+#   # -> ARROW_STATIC_LIBRARY_NAME=libarrow.dll.a with MinGW on Windows
+function(arrow_build_static_library_name output_variable base_name)
+  set(
+    ${output_variable}
+    "${CMAKE_STATIC_LIBRARY_PREFIX}${base_name}${ARROW_MSVC_STATIC_LIB_SUFFIX}${CMAKE_STATIC_LIBRARY_SUFFIX}"
+    PARENT_SCOPE)
+endfunction()
+
+# Internal function.
+#
+# Set macro value for ${macro_name} in ${header_content} to ${output_variable}.
+#
+# Example:
+#   arrow_extract_macro_value(version_major
+#                             "ARROW_VERSION_MAJOR"
+#                             "#define ARROW_VERSION_MAJOR 1.0.0")
+#   # -> version_major=1.0.0
+function(arrow_extract_macro_value output_variable macro_name header_content)
+  string(REGEX MATCH "#define +${macro_name} +[^\r\n]+" macro_definition
+               "${header_content}")
+  string(REGEX
+         REPLACE "^#define +${macro_name} +(.+)$" "\\1" macro_value "${macro_definition}")
+  set(${output_variable} "${macro_value}" PARENT_SCOPE)
+endfunction()
+
+# Internal macro only for arrow_find_package.
+#
+# Find package in HOME.
+macro(arrow_find_package_home)
+  find_path(${prefix}_include_dir "${header_path}"
+            PATHS "${home}"
+            PATH_SUFFIXES "include"
+            NO_DEFAULT_PATH)
+  set(include_dir "${${prefix}_include_dir}")
+  set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE)
+
+  if(MSVC)
+    set(CMAKE_SHARED_LIBRARY_SUFFIXES_ORIGINAL ${CMAKE_FIND_LIBRARY_SUFFIXES})
+    # .dll isn't found by find_library with MSVC because .dll isn't included in
+    # CMAKE_FIND_LIBRARY_SUFFIXES.
+    list(APPEND CMAKE_FIND_LIBRARY_SUFFIXES "${CMAKE_SHARED_LIBRARY_SUFFIX}")
+  endif()
+  find_library(${prefix}_shared_lib
+               NAMES "${shared_lib_name}"
+               PATHS "${home}"
+               PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES}
+               NO_DEFAULT_PATH)
+  if(MSVC)
+    set(CMAKE_SHARED_LIBRARY_SUFFIXES ${CMAKE_FIND_LIBRARY_SUFFIXES_ORIGINAL})
+  endif()
+  set(shared_lib "${${prefix}_shared_lib}")
+  set(${prefix}_SHARED_LIB "${shared_lib}" PARENT_SCOPE)
+  if(shared_lib)
+    add_library(${target_shared} SHARED IMPORTED)
+    set_target_properties(${target_shared} PROPERTIES IMPORTED_LOCATION "${shared_lib}")
+    if(include_dir)
+      set_target_properties(${target_shared}
+                            PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}")
+    endif()
+    find_library(${prefix}_import_lib
+                 NAMES "${import_lib_name}"
+                 PATHS "${home}"
+                 PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES}
+                 NO_DEFAULT_PATH)
+    set(import_lib "${${prefix}_import_lib}")
+    set(${prefix}_IMPORT_LIB "${import_lib}" PARENT_SCOPE)
+    if(import_lib)
+      set_target_properties(${target_shared} PROPERTIES IMPORTED_IMPLIB "${import_lib}")
+    endif()
+  endif()
+
+  find_library(${prefix}_static_lib
+               NAMES "${static_lib_name}"
+               PATHS "${home}"
+               PATH_SUFFIXES ${ARROW_SEARCH_LIB_PATH_SUFFIXES}
+               NO_DEFAULT_PATH)
+  set(static_lib "${${prefix}_static_lib}")
+  set(${prefix}_STATIC_LIB "${static_lib}" PARENT_SCOPE)
+  if(static_lib)
+    add_library(${target_static} STATIC IMPORTED)
+    set_target_properties(${target_static} PROPERTIES IMPORTED_LOCATION "${static_lib}")
+    if(include_dir)
+      set_target_properties(${target_static}
+                            PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}")
+    endif()
+  endif()
+endmacro()
+
+# Internal macro only for arrow_find_package.
+#
+# Find package by CMake package configuration.
+macro(arrow_find_package_cmake_package_configuration)
+  # ARROW-5575: We need to split target files for each component
+  if(TARGET ${target_shared} OR TARGET ${target_static})
+    set(${cmake_package_name}_FOUND TRUE)
+  else()
+    find_package(${cmake_package_name} CONFIG)
+  endif()
+  if(${cmake_package_name}_FOUND)
+    set(${prefix}_USE_CMAKE_PACKAGE_CONFIG TRUE PARENT_SCOPE)
+    if(TARGET ${target_shared})
+      foreach(suffix ${ARROW_CONFIG_SUFFIXES})
+        get_target_property(shared_lib ${target_shared} IMPORTED_LOCATION${suffix})
+        if(shared_lib)
+          # Remove shared library version:
+          #   libarrow.so.100.0.0 -> libarrow.so
+          # Because ARROW_HOME and pkg-config approaches don't add
+          # shared library version.
+          string(REGEX
+                 REPLACE "(${CMAKE_SHARED_LIBRARY_SUFFIX})[.0-9]+$" "\\1" shared_lib
+                         "${shared_lib}")
+          set(${prefix}_SHARED_LIB "${shared_lib}" PARENT_SCOPE)
+          break()
+        endif()
+      endforeach()
+    endif()
+    if(TARGET ${target_static})
+      foreach(suffix ${ARROW_CONFIG_SUFFIXES})
+        get_target_property(static_lib ${target_static} IMPORTED_LOCATION${suffix})
+        if(static_lib)
+          set(${prefix}_STATIC_LIB "${static_lib}" PARENT_SCOPE)
+          break()
+        endif()
+      endforeach()
+    endif()
+  endif()
+endmacro()
+
+# Internal macro only for arrow_find_package.
+#
+# Find package by pkg-config.
+macro(arrow_find_package_pkg_config)
+  pkg_check_modules(${prefix}_PC ${pkg_config_name})
+  if(${prefix}_PC_FOUND)
+    set(${prefix}_USE_PKG_CONFIG TRUE PARENT_SCOPE)
+
+    set(include_dir "${${prefix}_PC_INCLUDEDIR}")
+    set(lib_dir "${${prefix}_PC_LIBDIR}")
+    set(shared_lib_paths "${${prefix}_PC_LINK_LIBRARIES}")
+    # Use the first shared library path as the IMPORTED_LOCATION
+    # for ${target_shared}. This assumes that the first shared library
+    # path is the shared library path for this module.
+    list(GET shared_lib_paths 0 first_shared_lib_path)
+    # Use the rest shared library paths as the INTERFACE_LINK_LIBRARIES
+    # for ${target_shared}. This assumes that the rest shared library
+    # paths are dependency library paths for this module.
+    list(LENGTH shared_lib_paths n_shared_lib_paths)
+    if(n_shared_lib_paths LESS_EQUAL 1)
+      set(rest_shared_lib_paths)
+    else()
+      list(SUBLIST
+           shared_lib_paths
+           1
+           -1
+           rest_shared_lib_paths)
+    endif()
+
+    set(${prefix}_VERSION "${${prefix}_PC_VERSION}" PARENT_SCOPE)
+    set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE)
+    set(${prefix}_SHARED_LIB "${first_shared_lib_path}" PARENT_SCOPE)
+
+    add_library(${target_shared} SHARED IMPORTED)
+    set_target_properties(${target_shared}
+                          PROPERTIES INTERFACE_INCLUDE_DIRECTORIES
+                                     "${include_dir}"
+                                     INTERFACE_LINK_LIBRARIES
+                                     "${rest_shared_lib_paths}"
+                                     IMPORTED_LOCATION
+                                     "${first_shared_lib_path}")
+
+    find_library(${prefix}_static_lib
+                 NAMES "${static_lib_name}"
+                 PATHS "${lib_dir}"
+                 NO_DEFAULT_PATH)
+    set(static_lib "${${prefix}_static_lib}")
+    set(${prefix}_STATIC_LIB "${static_lib}" PARENT_SCOPE)
+    if(static_lib)
+      add_library(${target_static} STATIC IMPORTED)
+      set_target_properties(${target_static}
+                            PROPERTIES INTERFACE_INCLUDE_DIRECTORIES "${include_dir}"
+                                       IMPORTED_LOCATION "${static_lib}")
+    endif()
+  endif()
+endmacro()
+
+function(arrow_find_package
+         prefix
+         home
+         base_name
+         header_path
+         cmake_package_name
+         pkg_config_name)
+  arrow_build_shared_library_name(shared_lib_name ${base_name})
+  arrow_build_import_library_name(import_lib_name ${base_name})
+  arrow_build_static_library_name(static_lib_name ${base_name})
+
+  set(target_shared ${base_name}_shared)
+  set(target_static ${base_name}_static)
+
+  if(home)
+    arrow_find_package_home()
+    set(${prefix}_FIND_APPROACH "HOME: ${home}" PARENT_SCOPE)
+  else()
+    arrow_find_package_cmake_package_configuration()
+    if(${cmake_package_name}_FOUND)
+      set(${prefix}_FIND_APPROACH
+          "CMake package configuration: ${cmake_package_name}"
+          PARENT_SCOPE)
+    else()
+      arrow_find_package_pkg_config()
+      set(${prefix}_FIND_APPROACH "pkg-config: ${pkg_config_name}" PARENT_SCOPE)
+    endif()
+  endif()
+
+  if(NOT include_dir)
+    if(TARGET ${target_shared})
+      get_target_property(include_dir ${target_shared} INTERFACE_INCLUDE_DIRECTORIES)
+    elseif(TARGET ${target_static})
+      get_target_property(include_dir ${target_static} INTERFACE_INCLUDE_DIRECTORIES)
+    endif()
+  endif()
+  if(include_dir)
+    set(${prefix}_INCLUDE_DIR "${include_dir}" PARENT_SCOPE)
+  endif()
+
+  if(shared_lib)
+    get_filename_component(lib_dir "${shared_lib}" DIRECTORY)
+  elseif(static_lib)
+    get_filename_component(lib_dir "${static_lib}" DIRECTORY)
+  else()
+    set(lib_dir NOTFOUND)
+  endif()
+  set(${prefix}_LIB_DIR "${lib_dir}" PARENT_SCOPE)
+  # For backward compatibility
+  set(${prefix}_LIBS "${lib_dir}" PARENT_SCOPE)
+endfunction()
+
+if(NOT "$ENV{ARROW_HOME}" STREQUAL "")
+  file(TO_CMAKE_PATH "$ENV{ARROW_HOME}" ARROW_HOME)
+endif()
+arrow_find_package(ARROW
+                   "${ARROW_HOME}"
+                   arrow
+                   arrow/api.h
+                   Arrow
+                   arrow)
+
+if(ARROW_HOME)
+  if(ARROW_INCLUDE_DIR)
+    file(READ "${ARROW_INCLUDE_DIR}/arrow/util/config.h" ARROW_CONFIG_H_CONTENT)
+    arrow_extract_macro_value(ARROW_VERSION_MAJOR "ARROW_VERSION_MAJOR"
+                              "${ARROW_CONFIG_H_CONTENT}")
+    arrow_extract_macro_value(ARROW_VERSION_MINOR "ARROW_VERSION_MINOR"
+                              "${ARROW_CONFIG_H_CONTENT}")
+    arrow_extract_macro_value(ARROW_VERSION_PATCH "ARROW_VERSION_PATCH"
+                              "${ARROW_CONFIG_H_CONTENT}")
+    if("${ARROW_VERSION_MAJOR}" STREQUAL ""
+       OR "${ARROW_VERSION_MINOR}" STREQUAL ""
+       OR "${ARROW_VERSION_PATCH}" STREQUAL "")
+      set(ARROW_VERSION "0.0.0")
+    else()
+      set(ARROW_VERSION
+          "${ARROW_VERSION_MAJOR}.${ARROW_VERSION_MINOR}.${ARROW_VERSION_PATCH}")
+    endif()
+
+    arrow_extract_macro_value(ARROW_SO_VERSION_QUOTED "ARROW_SO_VERSION"
+                              "${ARROW_CONFIG_H_CONTENT}")
+    string(REGEX REPLACE "^\"(.+)\"$" "\\1" ARROW_SO_VERSION "${ARROW_SO_VERSION_QUOTED}")
+    arrow_extract_macro_value(ARROW_FULL_SO_VERSION_QUOTED "ARROW_FULL_SO_VERSION"
+                              "${ARROW_CONFIG_H_CONTENT}")
+    string(REGEX
+           REPLACE "^\"(.+)\"$" "\\1" ARROW_FULL_SO_VERSION
+                   "${ARROW_FULL_SO_VERSION_QUOTED}")
+  endif()
+else()
+  if(ARROW_USE_CMAKE_PACKAGE_CONFIG)
+    find_package(Arrow CONFIG)
+  elseif(ARROW_USE_PKG_CONFIG)
+    pkg_get_variable(ARROW_SO_VERSION arrow so_version)
+    pkg_get_variable(ARROW_FULL_SO_VERSION arrow full_so_version)
+  endif()
+endif()
+
+set(ARROW_ABI_VERSION ${ARROW_SO_VERSION})
+
+mark_as_advanced(ARROW_ABI_VERSION
+                 ARROW_CONFIG_SUFFIXES
+                 ARROW_FULL_SO_VERSION
+                 ARROW_IMPORT_LIB
+                 ARROW_INCLUDE_DIR
+                 ARROW_LIBS
+                 ARROW_LIB_DIR
+                 ARROW_SEARCH_LIB_PATH_SUFFIXES
+                 ARROW_SHARED_IMP_LIB
+                 ARROW_SHARED_LIB
+                 ARROW_SO_VERSION
+                 ARROW_STATIC_LIB
+                 ARROW_VERSION
+                 ARROW_VERSION_MAJOR
+                 ARROW_VERSION_MINOR
+                 ARROW_VERSION_PATCH)
+
+find_package_handle_standard_args(Arrow REQUIRED_VARS
+                                  # The first required variable is shown
+                                  # in the found message. So this list is
+                                  # not sorted alphabetically.
+                                  ARROW_INCLUDE_DIR
+                                  ARROW_LIB_DIR
+                                  ARROW_FULL_SO_VERSION
+                                  ARROW_SO_VERSION
+                                  VERSION_VAR
+                                  ARROW_VERSION)
+set(ARROW_FOUND ${Arrow_FOUND})
+
+if(Arrow_FOUND AND NOT Arrow_FIND_QUIETLY)
+  message(STATUS "Arrow version: ${ARROW_VERSION} (${ARROW_FIND_APPROACH})")
+  message(STATUS "Arrow SO and ABI version: ${ARROW_SO_VERSION}")
+  message(STATUS "Arrow full SO version: ${ARROW_FULL_SO_VERSION}")
+  message(STATUS "Found the Arrow core shared library: ${ARROW_SHARED_LIB}")
+  message(STATUS "Found the Arrow core import library: ${ARROW_IMPORT_LIB}")
+  message(STATUS "Found the Arrow core static library: ${ARROW_STATIC_LIB}")
+endif()

core/src/index/cmake/ThirdPartyPackagesCore.cmake

@@ -16,7 +16,7 @@
 
 set(KNOWHERE_THIRDPARTY_DEPENDENCIES
 
-        ARROW
+        Arrow
         FAISS
         GTest
         LAPACK
@@ -33,7 +33,7 @@ foreach (DEPENDENCY ${KNOWHERE_THIRDPARTY_DEPENDENCIES})
 endforeach ()
 
 macro(build_dependency DEPENDENCY_NAME)
-    if ("${DEPENDENCY_NAME}" STREQUAL "ARROW")
+    if ("${DEPENDENCY_NAME}" STREQUAL "Arrow")
         build_arrow()
     elseif ("${DEPENDENCY_NAME}" STREQUAL "LAPACK")
         build_lapack()
@@ -50,9 +50,10 @@ endmacro()
 
 macro(resolve_dependency DEPENDENCY_NAME)
     if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO")
-        #message(STATUS "Finding ${DEPENDENCY_NAME} package")
-        #message(STATUS "${DEPENDENCY_NAME} package not found")
-        build_dependency(${DEPENDENCY_NAME})
+        find_package(${DEPENDENCY_NAME} MODULE)
+        if(NOT ${${DEPENDENCY_NAME}_FOUND})
+          build_dependency(${DEPENDENCY_NAME})
+        endif()
     elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED")
         build_dependency(${DEPENDENCY_NAME})
     elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM")
@@ -131,17 +132,6 @@ if (USE_JFROG_CACHE STREQUAL "ON")
     endif ()
 endif ()
 
-macro(resolve_dependency DEPENDENCY_NAME)
-    if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO")
-        #disable find_package for now
-        build_dependency(${DEPENDENCY_NAME})
-    elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "BUNDLED")
-        build_dependency(${DEPENDENCY_NAME})
-    elseif (${DEPENDENCY_NAME}_SOURCE STREQUAL "SYSTEM")
-        find_package(${DEPENDENCY_NAME} REQUIRED)
-    endif ()
-endmacro()
-
 # ----------------------------------------------------------------------
 # ExternalProject options
 
@@ -296,6 +286,7 @@ macro(build_arrow)
     set(ARROW_STATIC_LIB
             "${ARROW_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}${ARROW_STATIC_LIB_NAME}${CMAKE_STATIC_LIBRARY_SUFFIX}"
             )
+    set(ARROW_LIB_DIR "${ARROW_PREFIX}/lib")
     set(ARROW_INCLUDE_DIR "${ARROW_PREFIX}/include")
 
     set(ARROW_CMAKE_ARGS
@@ -392,7 +383,7 @@ macro(build_arrow)
                 )
     endif ()
 
-    file(MAKE_DIRECTORY "${ARROW_PREFIX}/include")
+    file(MAKE_DIRECTORY "${ARROW_INCLUDE_DIR}")
     add_library(arrow STATIC IMPORTED)
     set_target_properties(arrow
             PROPERTIES IMPORTED_LOCATION "${ARROW_STATIC_LIB}"
@@ -402,17 +393,17 @@ macro(build_arrow)
     set(JEMALLOC_PREFIX "${INDEX_BINARY_DIR}/arrow_ep-prefix/src/arrow_ep-build/jemalloc_ep-prefix/src/jemalloc_ep")
 
     add_custom_command(TARGET arrow_ep POST_BUILD
-            COMMAND ${CMAKE_COMMAND} -E make_directory ${ARROW_PREFIX}/lib/
-            COMMAND ${CMAKE_COMMAND} -E copy ${JEMALLOC_PREFIX}/lib/libjemalloc_pic.a ${ARROW_PREFIX}/lib/
+            COMMAND ${CMAKE_COMMAND} -E make_directory ${ARROW_LIB_DIR}
+            COMMAND ${CMAKE_COMMAND} -E copy ${JEMALLOC_PREFIX}/lib/libjemalloc_pic.a ${ARROW_LIB_DIR}
             DEPENDS ${JEMALLOC_PREFIX}/lib/libjemalloc_pic.a)
 
 endmacro()
 
 if (KNOWHERE_WITH_ARROW AND NOT TARGET arrow_ep)
 
-    resolve_dependency(ARROW)
+    resolve_dependency(Arrow)
 
-    link_directories(SYSTEM ${ARROW_PREFIX}/lib/)
+    link_directories(SYSTEM ${ARROW_LIB_DIR})
     include_directories(SYSTEM ${ARROW_INCLUDE_DIR})
 endif ()
 

core/src/index/knowhere/CMakeLists.txt

@@ -49,7 +49,7 @@ set(depend_libs
         SPTAGLibStatic
         faiss
         arrow
-        ${ARROW_PREFIX}/lib/libjemalloc_pic.a
+        ${ARROW_LIB_DIR}/libjemalloc_pic.a
         gomp
         gfortran
         pthread

core/src/index/thirdparty/versions.txt

@@ -1,6 +1,7 @@
-ARROW_VERSION=apache-arrow-0.14.0
+ARROW_VERSION=apache-arrow-0.15.1
 BOOST_VERSION=1.70.0
 GTEST_VERSION=1.8.1
 LAPACK_VERSION=v3.8.0
 OPENBLAS_VERSION=v0.3.6
-FAISS_VERSION=branch-0.3.0
+FAISS_VERSION=branch-0.3.0
+MKL_VERSION=2019.5.281

core/src/index/unittest/CMakeLists.txt

@@ -6,7 +6,7 @@ include_directories(${INDEX_SOURCE_DIR})
 set(depend_libs
         gtest gmock gtest_main gmock_main
         faiss
-        arrow "${ARROW_PREFIX}/lib/libjemalloc_pic.a"
+        arrow "${ARROW_LIB_DIR}/libjemalloc_pic.a"
         )
 if (BUILD_FAISS_WITH_MKL)
     set(depend_libs ${depend_libs}

core/src/index/unittest/faiss_benchmark/CMakeLists.txt

@@ -13,7 +13,7 @@ if (KNOWHERE_GPU_VERSION)
 
     set(depend_libs
             faiss hdf5
-            arrow ${ARROW_PREFIX}/lib/libjemalloc_pic.a
+            arrow ${ARROW_LIB_DIR}/libjemalloc_pic.a
             )
     if (BUILD_FAISS_WITH_MKL)
         set(depend_libs ${depend_libs}

core/src/index/unittest/faiss_ori/CMakeLists.txt

@@ -8,7 +8,7 @@ if (KNOWHERE_GPU_VERSION)
 
     set(depend_libs
             faiss
-            arrow ${ARROW_PREFIX}/lib/libjemalloc_pic.a
+            arrow ${ARROW_LIB_DIR}/libjemalloc_pic.a
             )
     if (BUILD_FAISS_WITH_MKL)
         set(depend_libs ${depend_libs}

core/thirdparty/versions.txt

@@ -9,6 +9,5 @@ LIBUNWIND_VERSION=1.3.1
 GPERFTOOLS_VERSION=2.7
 GRPC_VERSION=master
 ZLIB_VERSION=v1.2.11
-MKL_VERSION=2019.5.281
 
 # vim: set filetype=sh: