From c5ce84db20fbee3f843ea1e28b70d7acee5061d1 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Thu, 1 Aug 2019 17:34:57 +0800 Subject: [PATCH 1/5] add third party cache function Former-commit-id: ef521babd2c0a53bb42da74ef57c8ec0dd50d251 --- cpp/cmake/BuildUtils.cmake | 57 ++++++++++++++++- cpp/cmake/ThirdPartyPackages.cmake | 99 +++++++++++++++++++++++------- 2 files changed, 133 insertions(+), 23 deletions(-) diff --git a/cpp/cmake/BuildUtils.cmake b/cpp/cmake/BuildUtils.cmake index 361d038c22..867a7ce93f 100644 --- a/cpp/cmake/BuildUtils.cmake +++ b/cpp/cmake/BuildUtils.cmake @@ -1,3 +1,58 @@ +# Define a function that extracts a cached package +function(ExternalProject_Use_Cache project_name package_file install_path) + message(STATUS "Will use cached package file: ${package_file}") + + ExternalProject_Add(${project_name} + DOWNLOAD_COMMAND ${CMAKE_COMMAND} -E echo + "No download step needed (using cached package)" + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E echo + "No configure step needed (using cached package)" + BUILD_COMMAND ${CMAKE_COMMAND} -E echo + "No build step needed (using cached package)" + INSTALL_COMMAND ${CMAKE_COMMAND} -E echo + "No install step needed (using cached package)" + ) + + # We want our tar files to contain the Install/ prefix (not for any + # very special reason, only for consistency and so that we can identify them + # in the extraction logs) which means that we must extract them in the + # binary (top-level build) directory to have them installed in the right + # place for subsequent ExternalProjects to pick them up. It seems that the + # only way to control the working directory is with Add_Step! + ExternalProject_Add_Step(${project_name} extract + ALWAYS 1 + COMMAND + ${CMAKE_COMMAND} -E echo + "Extracting ${package_file} to ${install_path}" + COMMAND + ${CMAKE_COMMAND} -E tar xzvf ${package_file} ${install_path} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + + ExternalProject_Add_StepTargets(${project_name} extract) +endfunction() + +# Define a function that to create a new cached package +function(ExternalProject_Create_Cache project_name package_file install_path cache_username cache_password cache_path) + if(EXISTS ${package_file}) + message(STATUS "Removing existing package file: ${package_file}") + file(REMOVE ${package_file}) + endif() + + message(STATUS "Will create cached package file: ${package_file}") + + ExternalProject_Add_Step(${project_name} package + DEPENDEES install + BYPRODUCTS ${package_file} + COMMAND ${CMAKE_COMMAND} -E echo "Updating cached package file: ${package_file}" + COMMAND ${CMAKE_COMMAND} -E tar czvf ${package_file} ${install_path} + COMMAND ${CMAKE_COMMAND} -E echo "Uploading package file ${package_file} to ${cache_path}" + COMMAND curl -u${cache_username}:${cache_password} -T ${package_file} ${cache_path} + ) + + ExternalProject_Add_StepTargets(${project_name} package) +endfunction() + function(ADD_THIRDPARTY_LIB LIB_NAME) set(options) set(one_value_args SHARED_LIB STATIC_LIB) @@ -92,4 +147,4 @@ function(ADD_THIRDPARTY_LIB LIB_NAME) else() message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}") endif() -endfunction() \ No newline at end of file +endfunction() diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 2804d98d98..aa67a96801 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -102,6 +102,19 @@ macro(build_dependency DEPENDENCY_NAME) endif () endmacro() + +# ---------------------------------------------------------------------- +# JFrog +if(NOT DEFINED USE_JFROG_CACHE) + set(USE_JFROG_CACHE "ON") +endif() +if(USE_JFROG_CACHE STREQUAL "ON") + set(JFROG_ARTFACTORY_CACHE_URL "http://192.168.1.201:80/artifactory/generic-local/thirdparty/cache") + set(JFROG_USER_NAME "test") + set(JFROG_PASSWORD "Fantast1c") + set(THIRDPARTY_PACKAGE_CACHE "${THIRDPARTY_DIR}/cache") +endif() + macro(resolve_dependency DEPENDENCY_NAME) if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") #disable find_package for now @@ -224,6 +237,7 @@ if(DEFINED ENV{MILVUS_BZIP2_URL}) else() set(BZIP2_SOURCE_URL "https://sourceware.org/pub/bzip2/bzip2-${BZIP2_VERSION}.tar.gz") endif() +set(BZIP2_MD5 "00b516f4704d4a7cb50a1d97e6e8e15b") if(DEFINED ENV{MILVUS_EASYLOGGINGPP_URL}) set(EASYLOGGINGPP_SOURCE_URL "$ENV{MILVUS_EASYLOGGINGPP_URL}") @@ -538,29 +552,69 @@ macro(build_bzip2) set(BZIP2_STATIC_LIB "${BZIP2_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}bz2${CMAKE_STATIC_LIBRARY_SUFFIX}") - externalproject_add(bzip2_ep - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "" - BUILD_IN_SOURCE - 1 - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - CFLAGS=${EP_C_FLAGS} - INSTALL_COMMAND - ${MAKE} - install - PREFIX=${BZIP2_PREFIX} - CFLAGS=${EP_C_FLAGS} - INSTALL_DIR - ${BZIP2_PREFIX} - URL - ${BZIP2_SOURCE_URL} - BUILD_BYPRODUCTS - "${BZIP2_STATIC_LIB}") + set(BZIP2_CACHE_PACKAGE_NAME "bzip2_${BZIP2_MD5}.tar.gz") + set(BZIP2_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${BZIP2_CACHE_PACKAGE_NAME}") + set(BZIP2_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${BZIP2_CACHE_PACKAGE_NAME}") - file(MAKE_DIRECTORY "${BZIP2_INCLUDE_DIR}") + if(USE_JFROG_CACHE STREQUAL "ON") + file(DOWNLOAD ${BZIP2_CACHE_URL} ${BZIP2_CACHE_PACKAGE_PATH} STATUS status) + list(GET status 0 status_code) + message(STATUS "DOWNLOADING FROM ${BZIP2_CACHE_URL} TO ${BZIP2_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") + if (NOT status_code EQUAL 0) + externalproject_add(bzip2_ep + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "" + BUILD_IN_SOURCE + 1 + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + CFLAGS=${EP_C_FLAGS} + INSTALL_COMMAND + ${MAKE} + install + PREFIX=${BZIP2_PREFIX} + CFLAGS=${EP_C_FLAGS} + INSTALL_DIR + ${BZIP2_PREFIX} + URL + ${BZIP2_SOURCE_URL} + BUILD_BYPRODUCTS + "${BZIP2_STATIC_LIB}") + + ExternalProject_Create_Cache(bzip2_ep ${BZIP2_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/bzip2_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${BZIP2_CACHE_URL}) + + file(MAKE_DIRECTORY "${BZIP2_INCLUDE_DIR}") + + else() + ExternalProject_Use_Cache(bzip2_ep ${BZIP2_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) + endif() + else() + externalproject_add(bzip2_ep + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "" + BUILD_IN_SOURCE + 1 + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + CFLAGS=${EP_C_FLAGS} + INSTALL_COMMAND + ${MAKE} + install + PREFIX=${BZIP2_PREFIX} + CFLAGS=${EP_C_FLAGS} + INSTALL_DIR + ${BZIP2_PREFIX} + URL + ${BZIP2_SOURCE_URL} + BUILD_BYPRODUCTS + "${BZIP2_STATIC_LIB}") + file(MAKE_DIRECTORY "${BZIP2_INCLUDE_DIR}") + endif() + add_library(bzip2 STATIC IMPORTED) set_target_properties( bzip2 @@ -1753,3 +1807,4 @@ if(MILVUS_WITH_GPERFTOOLS) include_directories(SYSTEM ${GPERFTOOLS_INCLUDE_DIR}) link_directories(SYSTEM ${GPERFTOOLS_PREFIX}/lib) endif() + From 932f665a6a2e3e5d138ca0cc5ddd0671f11dca10 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Thu, 1 Aug 2019 19:03:21 +0800 Subject: [PATCH 2/5] add boost & easylogging cache Former-commit-id: 19e1f06da655df2ad9dee0088e283af5b811a364 --- cpp/cmake/ThirdPartyPackages.cmake | 187 ++++++++++++++++++++++++----- 1 file changed, 158 insertions(+), 29 deletions(-) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index aa67a96801..285a1296be 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -102,6 +102,57 @@ macro(build_dependency DEPENDENCY_NAME) endif () endmacro() +# ---------------------------------------------------------------------- +# Identify OS +if (UNIX) + if (APPLE) + set (CMAKE_OS_NAME "osx" CACHE STRING "Operating system name" FORCE) + else (APPLE) + ## Check for Debian GNU/Linux ________________ + find_file (DEBIAN_FOUND debian_version debconf.conf + PATHS /etc + ) + if (DEBIAN_FOUND) + set (CMAKE_OS_NAME "debian" CACHE STRING "Operating system name" FORCE) + endif (DEBIAN_FOUND) + ## Check for Fedora _________________________ + find_file (FEDORA_FOUND fedora-release + PATHS /etc + ) + if (FEDORA_FOUND) + set (CMAKE_OS_NAME "fedora" CACHE STRING "Operating system name" FORCE) + endif (FEDORA_FOUND) + ## Check for RedHat _________________________ + find_file (REDHAT_FOUND redhat-release inittab.RH + PATHS /etc + ) + if (REDHAT_FOUND) + set (CMAKE_OS_NAME "redhat" CACHE STRING "Operating system name" FORCE) + endif (REDHAT_FOUND) + ## Extra check for Ubuntu ____________________ + if (DEBIAN_FOUND) + ## At its core Ubuntu is a Debian system, with + ## a slightly altered configuration; hence from + ## a first superficial inspection a system will + ## be considered as Debian, which signifies an + ## extra check is required. + find_file (UBUNTU_EXTRA legal issue + PATHS /etc + ) + if (UBUNTU_EXTRA) + ## Scan contents of file + file (STRINGS ${UBUNTU_EXTRA} UBUNTU_FOUND + REGEX Ubuntu + ) + ## Check result of string search + if (UBUNTU_FOUND) + set (CMAKE_OS_NAME "ubuntu" CACHE STRING "Operating system name" FORCE) + set (DEBIAN_FOUND FALSE) + endif (UBUNTU_FOUND) + endif (UBUNTU_EXTRA) + endif (DEBIAN_FOUND) + endif (APPLE) +endif (UNIX) # ---------------------------------------------------------------------- # JFrog @@ -109,11 +160,11 @@ if(NOT DEFINED USE_JFROG_CACHE) set(USE_JFROG_CACHE "ON") endif() if(USE_JFROG_CACHE STREQUAL "ON") - set(JFROG_ARTFACTORY_CACHE_URL "http://192.168.1.201:80/artifactory/generic-local/thirdparty/cache") + set(JFROG_ARTFACTORY_CACHE_URL "http://192.168.1.201:80/artifactory/generic-local/thirdparty/cache/${CMAKE_OS_NAME}/${MILVUS_BUILD_ARCH}") set(JFROG_USER_NAME "test") set(JFROG_PASSWORD "Fantast1c") set(THIRDPARTY_PACKAGE_CACHE "${THIRDPARTY_DIR}/cache") -endif() +endif() macro(resolve_dependency DEPENDENCY_NAME) if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") @@ -231,6 +282,7 @@ else() "https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION_UNDERSCORES}.tar.gz" ) endif() +set(BOOST_MD5 "fea771fe8176828fabf9c09242ee8c26") if(DEFINED ENV{MILVUS_BZIP2_URL}) set(BZIP2_SOURCE_URL "$ENV{MILVUS_BZIP2_URL}") @@ -244,12 +296,14 @@ if(DEFINED ENV{MILVUS_EASYLOGGINGPP_URL}) else() set(EASYLOGGINGPP_SOURCE_URL "https://github.com/zuhd-org/easyloggingpp/archive/${EASYLOGGINGPP_VERSION}.tar.gz") endif() +set(EASYLOGGINGPP_MD5 "b78cd319db4be9b639927657b8aa7732") if(DEFINED ENV{MILVUS_FAISS_URL}) set(FAISS_SOURCE_URL "$ENV{MILVUS_FAISS_URL}") else() set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/${FAISS_VERSION}.tar.gz") endif() +set(FAISS_MD5 "0bc12737b23def156f6a1eb782050135") if(DEFINED ENV{MILVUS_KNOWHERE_URL}) set(KNOWHERE_SOURCE_URL "$ENV{MILVUS_KNOWHERE_URL}") @@ -276,18 +330,21 @@ if(DEFINED ENV{MILVUS_LAPACK_URL}) else() set(LAPACK_SOURCE_URL "https://github.com/Reference-LAPACK/lapack/archive/${LAPACK_VERSION}.tar.gz") endif() +set(LAPACK_MD5 "96591affdbf58c450d45c1daa540dbd2") if(DEFINED ENV{MILVUS_LZ4_URL}) set(LZ4_SOURCE_URL "$ENV{MILVUS_LZ4_URL}") else() set(LZ4_SOURCE_URL "https://github.com/lz4/lz4/archive/${LZ4_VERSION}.tar.gz") endif() +set(LZ4_MD5 "a80f28f2a2e5fe59ebfe8407f793da22") if(DEFINED ENV{MILVUS_MYSQLPP_URL}) set(MYSQLPP_SOURCE_URL "$ENV{MILVUS_MYSQLPP_URL}") else() set(MYSQLPP_SOURCE_URL "https://tangentsoft.com/mysqlpp/releases/mysql++-${MYSQLPP_VERSION}.tar.gz") endif() +set(MYSQLPP_MD5 "cda38b5ecc0117de91f7c42292dd1e79") if (DEFINED ENV{MILVUS_OPENBLAS_URL}) set(OPENBLAS_SOURCE_URL "$ENV{MILVUS_OPENBLAS_URL}") @@ -295,6 +352,7 @@ else () set(OPENBLAS_SOURCE_URL "https://github.com/xianyi/OpenBLAS/archive/${OPENBLAS_VERSION}.tar.gz") endif() +set(OPENBLAS_MD5 "8a110a25b819a4b94e8a9580702b6495") if (DEFINED ENV{MILVUS_PROMETHEUS_URL}) set(PROMETHEUS_SOURCE_URL "$ENV{PROMETHEUS_OPENBLAS_URL}") @@ -303,6 +361,7 @@ else () https://github.com/jupp0r/prometheus-cpp.git) endif() + if (DEFINED ENV{MILVUS_ROCKSDB_URL}) set(ROCKSDB_SOURCE_URL "$ENV{MILVUS_ROCKSDB_URL}") else () @@ -316,6 +375,7 @@ else() set(SNAPPY_SOURCE_URL "https://github.com/google/snappy/archive/${SNAPPY_VERSION}.tar.gz") endif() +set(SNAPPY_MD5 "ee9086291c9ae8deb4dac5e0b85bf54a") if(DEFINED ENV{MILVUS_SQLITE_URL}) set(SQLITE_SOURCE_URL "$ENV{MILVUS_SQLITE_URL}") @@ -323,6 +383,7 @@ else() set(SQLITE_SOURCE_URL "https://www.sqlite.org/2019/sqlite-autoconf-${SQLITE_VERSION}.tar.gz") endif() +set(SQLITE_MD5 "3c68eb400f8354605736cd55400e1572") if(DEFINED ENV{MILVUS_SQLITE_ORM_URL}) set(SQLITE_ORM_SOURCE_URL "$ENV{MILVUS_SQLITE_ORM_URL}") @@ -330,6 +391,7 @@ else() set(SQLITE_ORM_SOURCE_URL "https://github.com/fnc12/sqlite_orm/archive/${SQLITE_ORM_VERSION}.zip") endif() +set(SQLITE_ORM_MD5 "ba9a405a8a1421c093aa8ce988ff8598") if(DEFINED ENV{MILVUS_THRIFT_URL}) set(THRIFT_SOURCE_URL "$ENV{MILVUS_THRIFT_URL}") @@ -337,30 +399,35 @@ else() set(THRIFT_SOURCE_URL "https://github.com/apache/thrift/archive/${THRIFT_VERSION}.tar.gz") endif() +set(THRIFT_MD5 "ff9af01fec424b5a279fa8a3c9e95c0c") if(DEFINED ENV{MILVUS_YAMLCPP_URL}) set(YAMLCPP_SOURCE_URL "$ENV{MILVUS_YAMLCPP_URL}") else() set(YAMLCPP_SOURCE_URL "https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-${YAMLCPP_VERSION}.tar.gz") endif() +set(YAMLCPP_MD5 "5b943e9af0060d0811148b037449ef82") if(DEFINED ENV{MILVUS_ZLIB_URL}) set(ZLIB_SOURCE_URL "$ENV{MILVUS_ZLIB_URL}") else() set(ZLIB_SOURCE_URL "https://github.com/madler/zlib/archive/${ZLIB_VERSION}.tar.gz") endif() +set(ZLIB_MD5 "0095d2d2d1f3442ce1318336637b695f") if(DEFINED ENV{MILVUS_ZSTD_URL}) set(ZSTD_SOURCE_URL "$ENV{MILVUS_ZSTD_URL}") else() set(ZSTD_SOURCE_URL "https://github.com/facebook/zstd/archive/${ZSTD_VERSION}.tar.gz") endif() +set(ZSTD_MD5 "340c837db48354f8d5eafe74c6077120") if(DEFINED ENV{MILVUS_AWS_URL}) set(AWS_SOURCE_URL "$ENV{MILVUS_AWS_URL}") else() set(AWS_SOURCE_URL "https://github.com/aws/aws-sdk-cpp/archive/${AWS_VERSION}.tar.gz") endif() +set(AWS_MD5 "9217f5bc8bf23dea04f4466521c85fd9") if(DEFINED ENV{MILVUS_LIBUNWIND_URL}) set(LIBUNWIND_SOURCE_URL "$ENV{MILVUS_LIBUNWIND_URL}") @@ -517,20 +584,52 @@ if(MILVUS_BOOST_VENDORED) set(MILVUS_BOOST_LIBS ${BOOST_SYSTEM_LIBRARY} ${BOOST_FILESYSTEM_LIBRARY} ${BOOST_STATIC_SERIALIZATION_LIBRARY}) endif() - externalproject_add(boost_ep - URL - ${BOOST_SOURCE_URL} - BUILD_BYPRODUCTS - ${BOOST_BUILD_PRODUCTS} - BUILD_IN_SOURCE - 1 - CONFIGURE_COMMAND - ${BOOST_CONFIGURE_COMMAND} - BUILD_COMMAND - ${BOOST_BUILD_COMMAND} - INSTALL_COMMAND - "" - ${EP_LOG_OPTIONS}) + + set(BOOST_CACHE_PACKAGE_NAME "boost_${BOOST_MD5}.tar.gz") + set(BOOST_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${BOOST_CACHE_PACKAGE_NAME}") + set(BOOST_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${BOOST_CACHE_PACKAGE_NAME}") + + if(USE_JFROG_CACHE STREQUAL "ON") + file(DOWNLOAD ${BOOST_CACHE_URL} ${BOOST_CACHE_PACKAGE_PATH} STATUS status) + list(GET status 0 status_code) + message(STATUS "DOWNLOADING FROM ${BOOST_CACHE_URL} TO ${BOOST_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") + if (NOT status_code EQUAL 0) + externalproject_add(boost_ep + URL + ${BOOST_SOURCE_URL} + BUILD_BYPRODUCTS + ${BOOST_BUILD_PRODUCTS} + BUILD_IN_SOURCE + 1 + CONFIGURE_COMMAND + ${BOOST_CONFIGURE_COMMAND} + BUILD_COMMAND + ${BOOST_BUILD_COMMAND} + INSTALL_COMMAND + "" + ${EP_LOG_OPTIONS}) + + ExternalProject_Create_Cache(boost_ep ${BOOST_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/boost_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${BOOST_CACHE_URL}) + else() + ExternalProject_Use_Cache(boost_ep ${BOOST_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) + endif() + else() + externalproject_add(boost_ep + URL + ${BOOST_SOURCE_URL} + BUILD_BYPRODUCTS + ${BOOST_BUILD_PRODUCTS} + BUILD_IN_SOURCE + 1 + CONFIGURE_COMMAND + ${BOOST_CONFIGURE_COMMAND} + BUILD_COMMAND + ${BOOST_BUILD_COMMAND} + INSTALL_COMMAND + "" + ${EP_LOG_OPTIONS}) + endif() + set(Boost_INCLUDE_DIR "${BOOST_PREFIX}") set(Boost_INCLUDE_DIRS "${Boost_INCLUDE_DIR}") add_dependencies(boost_system_static boost_ep) @@ -703,19 +802,50 @@ macro(build_easyloggingpp) -Dtest=OFF -Dbuild_static_lib=ON) - externalproject_add(easyloggingpp_ep - URL - ${EASYLOGGINGPP_SOURCE_URL} - ${EP_LOG_OPTIONS} - CMAKE_ARGS - ${EASYLOGGINGPP_CMAKE_ARGS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_BYPRODUCTS - ${EASYLOGGINGPP_STATIC_LIB}) + set(EASYLOGGINGPP_CACHE_PACKAGE_NAME "easyloggingpp_${EASYLOGGINGPP_MD5}.tar.gz") + set(EASYLOGGINGPP_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${EASYLOGGINGPP_CACHE_PACKAGE_NAME}") + set(EASYLOGGINGPP_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${EASYLOGGINGPP_CACHE_PACKAGE_NAME}") + + if(USE_JFROG_CACHE STREQUAL "ON") + file(DOWNLOAD ${EASYLOGGINGPP_CACHE_URL} ${EASYLOGGINGPP_CACHE_PACKAGE_PATH} STATUS status) + list(GET status 0 status_code) + message(STATUS "DOWNLOADING FROM ${EASYLOGGINGPP_CACHE_URL} TO ${EASYLOGGINGPP_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") + if (NOT status_code EQUAL 0) + externalproject_add(easyloggingpp_ep + URL + ${EASYLOGGINGPP_SOURCE_URL} + ${EP_LOG_OPTIONS} + CMAKE_ARGS + ${EASYLOGGINGPP_CMAKE_ARGS} + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + BUILD_BYPRODUCTS + ${EASYLOGGINGPP_STATIC_LIB}) + + ExternalProject_Create_Cache(easyloggingpp_ep ${EASYLOGGINGPP_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/easyloggingpp_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${EASYLOGGINGPP_CACHE_URL}) + + file(MAKE_DIRECTORY "${EASYLOGGINGPP_INCLUDE_DIR}") + else() + ExternalProject_Use_Cache(easyloggingpp_ep ${EASYLOGGINGPP_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) + endif() + else() + externalproject_add(easyloggingpp_ep + URL + ${EASYLOGGINGPP_SOURCE_URL} + ${EP_LOG_OPTIONS} + CMAKE_ARGS + ${EASYLOGGINGPP_CMAKE_ARGS} + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + BUILD_BYPRODUCTS + ${EASYLOGGINGPP_STATIC_LIB}) + + file(MAKE_DIRECTORY "${EASYLOGGINGPP_INCLUDE_DIR}") + endif() + - file(MAKE_DIRECTORY "${EASYLOGGINGPP_INCLUDE_DIR}") add_library(easyloggingpp STATIC IMPORTED) set_target_properties( easyloggingpp @@ -1807,4 +1937,3 @@ if(MILVUS_WITH_GPERFTOOLS) include_directories(SYSTEM ${GPERFTOOLS_INCLUDE_DIR}) link_directories(SYSTEM ${GPERFTOOLS_PREFIX}/lib) endif() - From 969f780ff344a24d3152373b1cdab1368296d62d Mon Sep 17 00:00:00 2001 From: quicksilver Date: Thu, 1 Aug 2019 20:42:47 +0800 Subject: [PATCH 3/5] add openblas cache Former-commit-id: c686ffa55a33c18b459cfb659a86ae0badb7782e --- cpp/cmake/ThirdPartyPackages.cmake | 88 +++++++++++++++++++++--------- 1 file changed, 62 insertions(+), 26 deletions(-) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 285a1296be..640f0a7362 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -651,11 +651,11 @@ macro(build_bzip2) set(BZIP2_STATIC_LIB "${BZIP2_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}bz2${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(BZIP2_CACHE_PACKAGE_NAME "bzip2_${BZIP2_MD5}.tar.gz") - set(BZIP2_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${BZIP2_CACHE_PACKAGE_NAME}") - set(BZIP2_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${BZIP2_CACHE_PACKAGE_NAME}") - if(USE_JFROG_CACHE STREQUAL "ON") + set(BZIP2_CACHE_PACKAGE_NAME "bzip2_${BZIP2_MD5}.tar.gz") + set(BZIP2_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${BZIP2_CACHE_PACKAGE_NAME}") + set(BZIP2_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${BZIP2_CACHE_PACKAGE_NAME}") + file(DOWNLOAD ${BZIP2_CACHE_URL} ${BZIP2_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${BZIP2_CACHE_URL} TO ${BZIP2_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") @@ -802,11 +802,11 @@ macro(build_easyloggingpp) -Dtest=OFF -Dbuild_static_lib=ON) - set(EASYLOGGINGPP_CACHE_PACKAGE_NAME "easyloggingpp_${EASYLOGGINGPP_MD5}.tar.gz") - set(EASYLOGGINGPP_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${EASYLOGGINGPP_CACHE_PACKAGE_NAME}") - set(EASYLOGGINGPP_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${EASYLOGGINGPP_CACHE_PACKAGE_NAME}") - if(USE_JFROG_CACHE STREQUAL "ON") + set(EASYLOGGINGPP_CACHE_PACKAGE_NAME "easyloggingpp_${EASYLOGGINGPP_MD5}.tar.gz") + set(EASYLOGGINGPP_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${EASYLOGGINGPP_CACHE_PACKAGE_NAME}") + set(EASYLOGGINGPP_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${EASYLOGGINGPP_CACHE_PACKAGE_NAME}") + file(DOWNLOAD ${EASYLOGGINGPP_CACHE_URL} ${EASYLOGGINGPP_CACHE_PACKAGE_PATH} STATUS status) list(GET status 0 status_code) message(STATUS "DOWNLOADING FROM ${EASYLOGGINGPP_CACHE_URL} TO ${EASYLOGGINGPP_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") @@ -873,25 +873,61 @@ macro(build_openblas) set(OPENBLAS_STATIC_LIB "${OPENBLAS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}openblas${CMAKE_STATIC_LIBRARY_SUFFIX}") - externalproject_add(openblas_ep - URL - ${OPENBLAS_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "" - BUILD_IN_SOURCE - 1 - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - INSTALL_COMMAND - ${MAKE} - PREFIX=${OPENBLAS_PREFIX} - install - BUILD_BYPRODUCTS - ${OPENBLAS_STATIC_LIB}) + if(USE_JFROG_CACHE STREQUAL "ON") + set(OPENBLAS_CACHE_PACKAGE_NAME "openblas_${OPENBLAS_MD5}.tar.gz") + set(OPENBLAS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${OPENBLAS_CACHE_PACKAGE_NAME}") + set(OPENBLAS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${OPENBLAS_CACHE_PACKAGE_NAME}") + + file(DOWNLOAD ${OPENBLAS_CACHE_URL} ${OPENBLAS_CACHE_PACKAGE_PATH} STATUS status) + list(GET status 0 status_code) + message(STATUS "DOWNLOADING FROM ${OPENBLAS_CACHE_URL} TO ${OPENBLAS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") + if (NOT status_code EQUAL 0) + externalproject_add(openblas_ep + URL + ${OPENBLAS_SOURCE_URL} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "" + BUILD_IN_SOURCE + 1 + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + INSTALL_COMMAND + ${MAKE} + PREFIX=${OPENBLAS_PREFIX} + install + BUILD_BYPRODUCTS + ${OPENBLAS_STATIC_LIB}) + + ExternalProject_Create_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/openblas_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${OPENBLAS_CACHE_URL}) + + file(MAKE_DIRECTORY "${OPENBLAS_INCLUDE_DIR}") + else() + ExternalProject_Use_Cache(openblas_ep ${OPENBLAS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) + endif() + else() + externalproject_add(openblas_ep + URL + ${OPENBLAS_SOURCE_URL} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "" + BUILD_IN_SOURCE + 1 + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + INSTALL_COMMAND + ${MAKE} + PREFIX=${OPENBLAS_PREFIX} + install + BUILD_BYPRODUCTS + ${OPENBLAS_STATIC_LIB}) + + file(MAKE_DIRECTORY "${OPENBLAS_INCLUDE_DIR}") + endif() - file(MAKE_DIRECTORY "${OPENBLAS_INCLUDE_DIR}") add_library(openblas STATIC IMPORTED) set_target_properties( openblas From 3c2e45afa4a6cef96e87e7eef124137d61efca96 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Thu, 1 Aug 2019 20:46:19 +0800 Subject: [PATCH 4/5] add lapack cache Former-commit-id: 24c134fc2ddd18321e2503a2bda6b05e7d457b7e --- cpp/cmake/ThirdPartyPackages.cmake | 54 +++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 12 deletions(-) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 640f0a7362..625a002f4c 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -952,19 +952,49 @@ macro(build_lapack) "-DCMAKE_INSTALL_PREFIX=${LAPACK_PREFIX}" -DCMAKE_INSTALL_LIBDIR=lib) - externalproject_add(lapack_ep - URL - ${LAPACK_SOURCE_URL} - ${EP_LOG_OPTIONS} - CMAKE_ARGS - ${LAPACK_CMAKE_ARGS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_BYPRODUCTS - ${LAPACK_STATIC_LIB}) + if(USE_JFROG_CACHE STREQUAL "ON") + set(LAPACK_CACHE_PACKAGE_NAME "lapack_${LAPACK_MD5}.tar.gz") + set(LAPACK_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${LAPACK_CACHE_PACKAGE_NAME}") + set(LAPACK_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${LAPACK_CACHE_PACKAGE_NAME}") + + file(DOWNLOAD ${LAPACK_CACHE_URL} ${LAPACK_CACHE_PACKAGE_PATH} STATUS status) + list(GET status 0 status_code) + message(STATUS "DOWNLOADING FROM ${LAPACK_CACHE_URL} TO ${LAPACK_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") + if (NOT status_code EQUAL 0) + externalproject_add(lapack_ep + URL + ${LAPACK_SOURCE_URL} + ${EP_LOG_OPTIONS} + CMAKE_ARGS + ${LAPACK_CMAKE_ARGS} + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + BUILD_BYPRODUCTS + ${LAPACK_STATIC_LIB}) + + ExternalProject_Create_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/lapack_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${LAPACK_CACHE_URL}) + + file(MAKE_DIRECTORY "${LAPACK_INCLUDE_DIR}") + else() + ExternalProject_Use_Cache(lapack_ep ${LAPACK_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) + endif() + else() + externalproject_add(lapack_ep + URL + ${LAPACK_SOURCE_URL} + ${EP_LOG_OPTIONS} + CMAKE_ARGS + ${LAPACK_CMAKE_ARGS} + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + BUILD_BYPRODUCTS + ${LAPACK_STATIC_LIB}) + + file(MAKE_DIRECTORY "${LAPACK_INCLUDE_DIR}") + endif() - file(MAKE_DIRECTORY "${LAPACK_INCLUDE_DIR}") add_library(lapack STATIC IMPORTED) set_target_properties( lapack From dce58c03eacd0f3a0c6f678df25de5db622dfd46 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Thu, 1 Aug 2019 21:04:38 +0800 Subject: [PATCH 5/5] add faiss cache Former-commit-id: 39d09d89aa0f264fd5861c4dbc4794f90b99e9eb --- cpp/cmake/ThirdPartyPackages.cmake | 77 ++++++++++++++++++++++-------- 1 file changed, 58 insertions(+), 19 deletions(-) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 625a002f4c..43fe20b8bc 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -1066,27 +1066,66 @@ macro(build_faiss) set(FAISS_CONFIGURE_ARGS ${FAISS_CONFIGURE_ARGS} --without-cuda) endif() - externalproject_add(faiss_ep - URL - ${FAISS_SOURCE_URL} - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "./configure" - ${FAISS_CONFIGURE_ARGS} - BUILD_COMMAND - ${MAKE} ${MAKE_BUILD_ARGS} VERBOSE=1 - BUILD_IN_SOURCE - 1 - INSTALL_COMMAND - ${MAKE} install - BUILD_BYPRODUCTS - ${FAISS_STATIC_LIB}) - - if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF") - ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep) + if(USE_JFROG_CACHE STREQUAL "ON") + string(MD5 FAISS_COMBINE_MD5 "${FAISS_MD5}${LAPACK_MD5}${OPENBLAS_MD5}") + set(FAISS_CACHE_PACKAGE_NAME "faiss_${FAISS_COMBINE_MD5}.tar.gz") + set(FAISS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${FAISS_CACHE_PACKAGE_NAME}") + set(FAISS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${FAISS_CACHE_PACKAGE_NAME}") + + file(DOWNLOAD ${FAISS_CACHE_URL} ${FAISS_CACHE_PACKAGE_PATH} STATUS status) + list(GET status 0 status_code) + message(STATUS "DOWNLOADING FROM ${FAISS_CACHE_URL} TO ${FAISS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") + if (NOT status_code EQUAL 0) + externalproject_add(faiss_ep + URL + ${FAISS_SOURCE_URL} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "./configure" + ${FAISS_CONFIGURE_ARGS} + BUILD_COMMAND + ${MAKE} ${MAKE_BUILD_ARGS} VERBOSE=1 + BUILD_IN_SOURCE + 1 + INSTALL_COMMAND + ${MAKE} install + BUILD_BYPRODUCTS + ${FAISS_STATIC_LIB}) + + if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF") + ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep) + endif() + + ExternalProject_Create_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/faiss_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${FAISS_CACHE_URL}) + + file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}") + else() + ExternalProject_Use_Cache(faiss_ep ${FAISS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) + endif() + else() + externalproject_add(faiss_ep + URL + ${FAISS_SOURCE_URL} + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "./configure" + ${FAISS_CONFIGURE_ARGS} + BUILD_COMMAND + ${MAKE} ${MAKE_BUILD_ARGS} VERBOSE=1 + BUILD_IN_SOURCE + 1 + INSTALL_COMMAND + ${MAKE} install + BUILD_BYPRODUCTS + ${FAISS_STATIC_LIB}) + + if(${BUILD_FAISS_WITH_MKL} STREQUAL "OFF") + ExternalProject_Add_StepDependencies(faiss_ep build openblas_ep lapack_ep) + endif() + + file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}") endif() - file(MAKE_DIRECTORY "${FAISS_INCLUDE_DIR}") add_library(faiss SHARED IMPORTED) if(${BUILD_FAISS_WITH_MKL} STREQUAL "ON")