diff --git a/cpp/cmake/BuildUtils.cmake b/cpp/cmake/BuildUtils.cmake index 361d038c22..867a7ce93f 100644 --- a/cpp/cmake/BuildUtils.cmake +++ b/cpp/cmake/BuildUtils.cmake @@ -1,3 +1,58 @@ +# Define a function that extracts a cached package +function(ExternalProject_Use_Cache project_name package_file install_path) + message(STATUS "Will use cached package file: ${package_file}") + + ExternalProject_Add(${project_name} + DOWNLOAD_COMMAND ${CMAKE_COMMAND} -E echo + "No download step needed (using cached package)" + CONFIGURE_COMMAND ${CMAKE_COMMAND} -E echo + "No configure step needed (using cached package)" + BUILD_COMMAND ${CMAKE_COMMAND} -E echo + "No build step needed (using cached package)" + INSTALL_COMMAND ${CMAKE_COMMAND} -E echo + "No install step needed (using cached package)" + ) + + # We want our tar files to contain the Install/ prefix (not for any + # very special reason, only for consistency and so that we can identify them + # in the extraction logs) which means that we must extract them in the + # binary (top-level build) directory to have them installed in the right + # place for subsequent ExternalProjects to pick them up. It seems that the + # only way to control the working directory is with Add_Step! + ExternalProject_Add_Step(${project_name} extract + ALWAYS 1 + COMMAND + ${CMAKE_COMMAND} -E echo + "Extracting ${package_file} to ${install_path}" + COMMAND + ${CMAKE_COMMAND} -E tar xzvf ${package_file} ${install_path} + WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} + ) + + ExternalProject_Add_StepTargets(${project_name} extract) +endfunction() + +# Define a function that to create a new cached package +function(ExternalProject_Create_Cache project_name package_file install_path cache_username cache_password cache_path) + if(EXISTS ${package_file}) + message(STATUS "Removing existing package file: ${package_file}") + file(REMOVE ${package_file}) + endif() + + message(STATUS "Will create cached package file: ${package_file}") + + ExternalProject_Add_Step(${project_name} package + DEPENDEES install + BYPRODUCTS ${package_file} + COMMAND ${CMAKE_COMMAND} -E echo "Updating cached package file: ${package_file}" + COMMAND ${CMAKE_COMMAND} -E tar czvf ${package_file} ${install_path} + COMMAND ${CMAKE_COMMAND} -E echo "Uploading package file ${package_file} to ${cache_path}" + COMMAND curl -u${cache_username}:${cache_password} -T ${package_file} ${cache_path} + ) + + ExternalProject_Add_StepTargets(${project_name} package) +endfunction() + function(ADD_THIRDPARTY_LIB LIB_NAME) set(options) set(one_value_args SHARED_LIB STATIC_LIB) @@ -92,4 +147,4 @@ function(ADD_THIRDPARTY_LIB LIB_NAME) else() message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}") endif() -endfunction() \ No newline at end of file +endfunction() diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 2804d98d98..aa67a96801 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -102,6 +102,19 @@ macro(build_dependency DEPENDENCY_NAME) endif () endmacro() + +# ---------------------------------------------------------------------- +# JFrog +if(NOT DEFINED USE_JFROG_CACHE) + set(USE_JFROG_CACHE "ON") +endif() +if(USE_JFROG_CACHE STREQUAL "ON") + set(JFROG_ARTFACTORY_CACHE_URL "http://192.168.1.201:80/artifactory/generic-local/thirdparty/cache") + set(JFROG_USER_NAME "test") + set(JFROG_PASSWORD "Fantast1c") + set(THIRDPARTY_PACKAGE_CACHE "${THIRDPARTY_DIR}/cache") +endif() + macro(resolve_dependency DEPENDENCY_NAME) if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO") #disable find_package for now @@ -224,6 +237,7 @@ if(DEFINED ENV{MILVUS_BZIP2_URL}) else() set(BZIP2_SOURCE_URL "https://sourceware.org/pub/bzip2/bzip2-${BZIP2_VERSION}.tar.gz") endif() +set(BZIP2_MD5 "00b516f4704d4a7cb50a1d97e6e8e15b") if(DEFINED ENV{MILVUS_EASYLOGGINGPP_URL}) set(EASYLOGGINGPP_SOURCE_URL "$ENV{MILVUS_EASYLOGGINGPP_URL}") @@ -538,29 +552,69 @@ macro(build_bzip2) set(BZIP2_STATIC_LIB "${BZIP2_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}bz2${CMAKE_STATIC_LIBRARY_SUFFIX}") - externalproject_add(bzip2_ep - ${EP_LOG_OPTIONS} - CONFIGURE_COMMAND - "" - BUILD_IN_SOURCE - 1 - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - CFLAGS=${EP_C_FLAGS} - INSTALL_COMMAND - ${MAKE} - install - PREFIX=${BZIP2_PREFIX} - CFLAGS=${EP_C_FLAGS} - INSTALL_DIR - ${BZIP2_PREFIX} - URL - ${BZIP2_SOURCE_URL} - BUILD_BYPRODUCTS - "${BZIP2_STATIC_LIB}") + set(BZIP2_CACHE_PACKAGE_NAME "bzip2_${BZIP2_MD5}.tar.gz") + set(BZIP2_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${BZIP2_CACHE_PACKAGE_NAME}") + set(BZIP2_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${BZIP2_CACHE_PACKAGE_NAME}") - file(MAKE_DIRECTORY "${BZIP2_INCLUDE_DIR}") + if(USE_JFROG_CACHE STREQUAL "ON") + file(DOWNLOAD ${BZIP2_CACHE_URL} ${BZIP2_CACHE_PACKAGE_PATH} STATUS status) + list(GET status 0 status_code) + message(STATUS "DOWNLOADING FROM ${BZIP2_CACHE_URL} TO ${BZIP2_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") + if (NOT status_code EQUAL 0) + externalproject_add(bzip2_ep + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "" + BUILD_IN_SOURCE + 1 + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + CFLAGS=${EP_C_FLAGS} + INSTALL_COMMAND + ${MAKE} + install + PREFIX=${BZIP2_PREFIX} + CFLAGS=${EP_C_FLAGS} + INSTALL_DIR + ${BZIP2_PREFIX} + URL + ${BZIP2_SOURCE_URL} + BUILD_BYPRODUCTS + "${BZIP2_STATIC_LIB}") + + ExternalProject_Create_Cache(bzip2_ep ${BZIP2_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/bzip2_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${BZIP2_CACHE_URL}) + + file(MAKE_DIRECTORY "${BZIP2_INCLUDE_DIR}") + + else() + ExternalProject_Use_Cache(bzip2_ep ${BZIP2_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) + endif() + else() + externalproject_add(bzip2_ep + ${EP_LOG_OPTIONS} + CONFIGURE_COMMAND + "" + BUILD_IN_SOURCE + 1 + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + CFLAGS=${EP_C_FLAGS} + INSTALL_COMMAND + ${MAKE} + install + PREFIX=${BZIP2_PREFIX} + CFLAGS=${EP_C_FLAGS} + INSTALL_DIR + ${BZIP2_PREFIX} + URL + ${BZIP2_SOURCE_URL} + BUILD_BYPRODUCTS + "${BZIP2_STATIC_LIB}") + file(MAKE_DIRECTORY "${BZIP2_INCLUDE_DIR}") + endif() + add_library(bzip2 STATIC IMPORTED) set_target_properties( bzip2 @@ -1753,3 +1807,4 @@ if(MILVUS_WITH_GPERFTOOLS) include_directories(SYSTEM ${GPERFTOOLS_INCLUDE_DIR}) link_directories(SYSTEM ${GPERFTOOLS_PREFIX}/lib) endif() +