From fdfeb8590dcea1b3be1c768c2264f7cf01963d5c Mon Sep 17 00:00:00 2001 From: quicksilver Date: Thu, 1 Aug 2019 23:25:26 +0800 Subject: [PATCH 1/2] add zstd cache Former-commit-id: 84a380e4f0e9cb41a3f66a6dc874e67487f09cc6 --- cpp/cmake/ThirdPartyPackages.cmake | 69 ++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 99ca545557..fc4dd2d7cc 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -2033,7 +2033,7 @@ macro(build_zstd) set(ZSTD_STATIC_LIB "${ZSTD_PREFIX}/lib/libzstd.a") - + set(ZSTD_INCLUDE_DIR "${ZSTD_PREFIX}/include") set(ZSTD_CMAKE_ARGS ${ZSTD_CMAKE_ARGS} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} @@ -2045,28 +2045,61 @@ macro(build_zstd) message(FATAL_ERROR "Building zstd using ExternalProject requires at least CMake 3.7") endif() - externalproject_add(zstd_ep - ${EP_LOG_OPTIONS} - CMAKE_ARGS - ${ZSTD_CMAKE_ARGS} - SOURCE_SUBDIR - "build/cmake" - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - INSTALL_DIR - ${ZSTD_PREFIX} - URL - ${ZSTD_SOURCE_URL} - BUILD_BYPRODUCTS - "${ZSTD_STATIC_LIB}") + if(USE_JFROG_CACHE STREQUAL "ON") + set(ZSTD_CACHE_PACKAGE_NAME "zstd_${ZSTD_MD5}.tar.gz") + set(ZSTD_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${ZSTD_CACHE_PACKAGE_NAME}") + set(ZSTD_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${ZSTD_CACHE_PACKAGE_NAME}") - file(MAKE_DIRECTORY "${ZSTD_PREFIX}/include") + file(DOWNLOAD ${ZSTD_CACHE_URL} ${ZSTD_CACHE_PACKAGE_PATH} STATUS status) + list(GET status 0 status_code) + message(STATUS "DOWNLOADING FROM ${ZSTD_CACHE_URL} TO ${ZSTD_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") + if (NOT status_code EQUAL 0) + externalproject_add(zstd_ep + ${EP_LOG_OPTIONS} + CMAKE_ARGS + ${ZSTD_CMAKE_ARGS} + SOURCE_SUBDIR + "build/cmake" + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + INSTALL_DIR + ${ZSTD_PREFIX} + URL + ${ZSTD_SOURCE_URL} + BUILD_BYPRODUCTS + "${ZSTD_STATIC_LIB}") + + ExternalProject_Create_Cache(zstd_ep ${ZSTD_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/zstd_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${ZSTD_CACHE_URL}) + + file(MAKE_DIRECTORY "${ZSTD_INCLUDE_DIR}") + else() + ExternalProject_Use_Cache(zstd_ep ${ZSTD_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) + endif() + else() + externalproject_add(zstd_ep + ${EP_LOG_OPTIONS} + CMAKE_ARGS + ${ZSTD_CMAKE_ARGS} + SOURCE_SUBDIR + "build/cmake" + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + INSTALL_DIR + ${ZSTD_PREFIX} + URL + ${ZSTD_SOURCE_URL} + BUILD_BYPRODUCTS + "${ZSTD_STATIC_LIB}") + + file(MAKE_DIRECTORY "${ZSTD_INCLUDE_DIR}") + endif() add_library(zstd STATIC IMPORTED) set_target_properties(zstd PROPERTIES IMPORTED_LOCATION "${ZSTD_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${ZSTD_PREFIX}/include") + INTERFACE_INCLUDE_DIRECTORIES "${ZSTD_INCLUDE_DIR}") add_dependencies(zstd zstd_ep) endmacro() From 9d016c87968381d74ad2913319f8dc32d75dc584 Mon Sep 17 00:00:00 2001 From: quicksilver Date: Thu, 1 Aug 2019 23:30:46 +0800 Subject: [PATCH 2/2] add aws cache Former-commit-id: f95cfcd81aa4ffc728e2eedf22270f0ba7c0800c --- cpp/cmake/ThirdPartyPackages.cmake | 69 ++++++++++++++++++++++-------- 1 file changed, 51 insertions(+), 18 deletions(-) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index fc4dd2d7cc..754b147766 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -2133,7 +2133,7 @@ macro(build_aws) "${AWS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-core${CMAKE_STATIC_LIBRARY_SUFFIX}") set(AWS_CPP_SDK_S3_STATIC_LIB "${AWS_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}aws-cpp-sdk-s3${CMAKE_STATIC_LIBRARY_SUFFIX}") - + set(AWS_INCLUDE_DIR "${AWS_PREFIX}/include") set(AWS_CMAKE_ARGS ${AWS_CMAKE_ARGS} -DCMAKE_C_COMPILER=${CMAKE_C_COMPILER} @@ -2141,33 +2141,66 @@ macro(build_aws) -DCMAKE_C_FLAGS=${EP_C_FLAGS} -DCMAKE_CXX_FLAGS=${EP_CXX_FLAGS}) - externalproject_add(aws_ep - ${EP_LOG_OPTIONS} - CMAKE_ARGS - ${AWS_CMAKE_ARGS} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - INSTALL_DIR - ${AWS_PREFIX} - URL - ${AWS_SOURCE_URL} - BUILD_BYPRODUCTS - "${AWS_CPP_SDK_S3_STATIC_LIB}" - "${AWS_CPP_SDK_CORE_STATIC_LIB}") - file(MAKE_DIRECTORY "${AWS_PREFIX}/include") + if(USE_JFROG_CACHE STREQUAL "ON") + set(AWS_CACHE_PACKAGE_NAME "aws_${AWS_MD5}.tar.gz") + set(AWS_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${AWS_CACHE_PACKAGE_NAME}") + set(AWS_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${AWS_CACHE_PACKAGE_NAME}") + + file(DOWNLOAD ${AWS_CACHE_URL} ${AWS_CACHE_PACKAGE_PATH} STATUS status) + list(GET status 0 status_code) + message(STATUS "DOWNLOADING FROM ${AWS_CACHE_URL} TO ${AWS_CACHE_PACKAGE_PATH}. STATUS = ${status_code}") + if (NOT status_code EQUAL 0) + externalproject_add(aws_ep + ${EP_LOG_OPTIONS} + CMAKE_ARGS + ${AWS_CMAKE_ARGS} + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + INSTALL_DIR + ${AWS_PREFIX} + URL + ${AWS_SOURCE_URL} + BUILD_BYPRODUCTS + "${AWS_CPP_SDK_S3_STATIC_LIB}" + "${AWS_CPP_SDK_CORE_STATIC_LIB}") + + ExternalProject_Create_Cache(aws_ep ${AWS_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/aws_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${AWS_CACHE_URL}) + + file(MAKE_DIRECTORY "${AWS_INCLUDE_DIR}") + else() + ExternalProject_Use_Cache(aws_ep ${AWS_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR}) + endif() + else() + externalproject_add(aws_ep + ${EP_LOG_OPTIONS} + CMAKE_ARGS + ${AWS_CMAKE_ARGS} + BUILD_COMMAND + ${MAKE} + ${MAKE_BUILD_ARGS} + INSTALL_DIR + ${AWS_PREFIX} + URL + ${AWS_SOURCE_URL} + BUILD_BYPRODUCTS + "${AWS_CPP_SDK_S3_STATIC_LIB}" + "${AWS_CPP_SDK_CORE_STATIC_LIB}") + + file(MAKE_DIRECTORY "${AWS_INCLUDE_DIR}") + endif() add_library(aws-cpp-sdk-s3 STATIC IMPORTED) set_target_properties(aws-cpp-sdk-s3 PROPERTIES IMPORTED_LOCATION "${AWS_CPP_SDK_S3_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${AWS_PREFIX}/include" + INTERFACE_INCLUDE_DIRECTORIES "${AWS_INCLUDE_DIR}" INTERFACE_LINK_LIBRARIES "${AWS_PREFIX}/lib/libaws-c-event-stream.a;${AWS_PREFIX}/lib/libaws-checksums.a;${AWS_PREFIX}/lib/libaws-c-common.a") add_library(aws-cpp-sdk-core STATIC IMPORTED) set_target_properties(aws-cpp-sdk-core PROPERTIES IMPORTED_LOCATION "${AWS_CPP_SDK_CORE_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${AWS_PREFIX}/include" + INTERFACE_INCLUDE_DIRECTORIES "${AWS_INCLUDE_DIR}" INTERFACE_LINK_LIBRARIES "${AWS_PREFIX}/lib/libaws-c-event-stream.a;${AWS_PREFIX}/lib/libaws-checksums.a;${AWS_PREFIX}/lib/libaws-c-common.a") add_dependencies(aws-cpp-sdk-s3 aws_ep)