add third party cache function

Former-commit-id: ef521babd2c0a53bb42da74ef57c8ec0dd50d251

cpp/cmake/BuildUtils.cmake

@@ -1,3 +1,58 @@
+# Define a function that extracts a cached package
+function(ExternalProject_Use_Cache project_name package_file install_path)
+    message(STATUS "Will use cached package file: ${package_file}")
+
+    ExternalProject_Add(${project_name}
+        DOWNLOAD_COMMAND ${CMAKE_COMMAND} -E echo
+            "No download step needed (using cached package)"
+        CONFIGURE_COMMAND ${CMAKE_COMMAND} -E echo
+            "No configure step needed (using cached package)"
+        BUILD_COMMAND ${CMAKE_COMMAND} -E echo
+            "No build step needed (using cached package)"
+        INSTALL_COMMAND ${CMAKE_COMMAND} -E echo
+            "No install step needed (using cached package)"
+    )
+
+    # We want our tar files to contain the Install/<package> prefix (not for any
+    # very special reason, only for consistency and so that we can identify them
+    # in the extraction logs) which means that we must extract them in the
+    # binary (top-level build) directory to have them installed in the right
+    # place for subsequent ExternalProjects to pick them up. It seems that the
+    # only way to control the working directory is with Add_Step!
+    ExternalProject_Add_Step(${project_name} extract
+        ALWAYS 1
+        COMMAND
+            ${CMAKE_COMMAND} -E echo
+            "Extracting ${package_file} to ${install_path}"
+        COMMAND
+            ${CMAKE_COMMAND} -E tar xzvf ${package_file} ${install_path}
+            WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}
+    )
+
+    ExternalProject_Add_StepTargets(${project_name} extract)
+endfunction()
+
+# Define a function that to create a new cached package
+function(ExternalProject_Create_Cache project_name package_file install_path cache_username cache_password cache_path)
+    if(EXISTS ${package_file})
+        message(STATUS "Removing existing package file: ${package_file}")
+        file(REMOVE ${package_file})
+    endif()
+
+    message(STATUS "Will create cached package file: ${package_file}")
+
+    ExternalProject_Add_Step(${project_name} package
+        DEPENDEES install
+        BYPRODUCTS ${package_file}
+        COMMAND ${CMAKE_COMMAND} -E echo "Updating cached package file: ${package_file}"
+        COMMAND ${CMAKE_COMMAND} -E tar czvf ${package_file} ${install_path}
+        COMMAND ${CMAKE_COMMAND} -E echo "Uploading package file ${package_file} to ${cache_path}"
+        COMMAND curl -u${cache_username}:${cache_password} -T ${package_file} ${cache_path}
+    )
+
+    ExternalProject_Add_StepTargets(${project_name} package)
+endfunction()
+
 function(ADD_THIRDPARTY_LIB LIB_NAME)
     set(options)
     set(one_value_args SHARED_LIB STATIC_LIB)
@@ -92,4 +147,4 @@ function(ADD_THIRDPARTY_LIB LIB_NAME)
     else()
         message(FATAL_ERROR "No static or shared library provided for ${LIB_NAME}")
     endif()
-endfunction()
+endfunction()

cpp/cmake/ThirdPartyPackages.cmake

@@ -102,6 +102,19 @@ macro(build_dependency DEPENDENCY_NAME)
     endif ()
 endmacro()
 
+
+# ----------------------------------------------------------------------
+# JFrog
+if(NOT DEFINED USE_JFROG_CACHE)
+    set(USE_JFROG_CACHE "ON")
+endif()
+if(USE_JFROG_CACHE STREQUAL "ON")    
+    set(JFROG_ARTFACTORY_CACHE_URL "http://192.168.1.201:80/artifactory/generic-local/thirdparty/cache")
+    set(JFROG_USER_NAME "test")
+    set(JFROG_PASSWORD "Fantast1c")
+    set(THIRDPARTY_PACKAGE_CACHE "${THIRDPARTY_DIR}/cache")
+endif()    
+
 macro(resolve_dependency DEPENDENCY_NAME)
     if (${DEPENDENCY_NAME}_SOURCE STREQUAL "AUTO")
         #disable find_package for now
@@ -224,6 +237,7 @@ if(DEFINED ENV{MILVUS_BZIP2_URL})
 else()
     set(BZIP2_SOURCE_URL "https://sourceware.org/pub/bzip2/bzip2-${BZIP2_VERSION}.tar.gz")
 endif()
+set(BZIP2_MD5 "00b516f4704d4a7cb50a1d97e6e8e15b")
 
 if(DEFINED ENV{MILVUS_EASYLOGGINGPP_URL})
     set(EASYLOGGINGPP_SOURCE_URL "$ENV{MILVUS_EASYLOGGINGPP_URL}")
@@ -538,29 +552,69 @@ macro(build_bzip2)
     set(BZIP2_STATIC_LIB
             "${BZIP2_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}bz2${CMAKE_STATIC_LIBRARY_SUFFIX}")
 
-    externalproject_add(bzip2_ep
+    set(BZIP2_CACHE_PACKAGE_NAME "bzip2_${BZIP2_MD5}.tar.gz")
-                        ${EP_LOG_OPTIONS}
+    set(BZIP2_CACHE_URL "${JFROG_ARTFACTORY_CACHE_URL}/${BZIP2_CACHE_PACKAGE_NAME}")
-                        CONFIGURE_COMMAND
+    set(BZIP2_CACHE_PACKAGE_PATH "${THIRDPARTY_PACKAGE_CACHE}/${BZIP2_CACHE_PACKAGE_NAME}")
-                        ""
-                        BUILD_IN_SOURCE
-                        1
-                        BUILD_COMMAND
-                        ${MAKE}
-                        ${MAKE_BUILD_ARGS}
-                        CFLAGS=${EP_C_FLAGS}
-                        INSTALL_COMMAND
-                        ${MAKE}
-                        install
-                        PREFIX=${BZIP2_PREFIX}
-                        CFLAGS=${EP_C_FLAGS}
-                        INSTALL_DIR
-                        ${BZIP2_PREFIX}
-                        URL
-                        ${BZIP2_SOURCE_URL}
-                        BUILD_BYPRODUCTS
-                        "${BZIP2_STATIC_LIB}")
 
-    file(MAKE_DIRECTORY "${BZIP2_INCLUDE_DIR}")
+    if(USE_JFROG_CACHE STREQUAL "ON") 
+        file(DOWNLOAD ${BZIP2_CACHE_URL} ${BZIP2_CACHE_PACKAGE_PATH} STATUS status)
+        list(GET status 0 status_code)
+        message(STATUS "DOWNLOADING FROM ${BZIP2_CACHE_URL} TO ${BZIP2_CACHE_PACKAGE_PATH}. STATUS = ${status_code}")
+        if (NOT status_code EQUAL 0)
+            externalproject_add(bzip2_ep
+                                ${EP_LOG_OPTIONS}
+                                CONFIGURE_COMMAND
+                                ""
+                                BUILD_IN_SOURCE
+                                1
+                                BUILD_COMMAND
+                                ${MAKE}
+                                ${MAKE_BUILD_ARGS}
+                                CFLAGS=${EP_C_FLAGS}
+                                INSTALL_COMMAND
+                                ${MAKE}
+                                install
+                                PREFIX=${BZIP2_PREFIX}
+                                CFLAGS=${EP_C_FLAGS}
+                                INSTALL_DIR
+                                ${BZIP2_PREFIX}
+                                URL
+                                ${BZIP2_SOURCE_URL}
+                                BUILD_BYPRODUCTS
+                                "${BZIP2_STATIC_LIB}")
+
+            ExternalProject_Create_Cache(bzip2_ep ${BZIP2_CACHE_PACKAGE_PATH} "${CMAKE_CURRENT_BINARY_DIR}/bzip2_ep-prefix" ${JFROG_USER_NAME} ${JFROG_PASSWORD} ${BZIP2_CACHE_URL})
+
+            file(MAKE_DIRECTORY "${BZIP2_INCLUDE_DIR}")
+
+        else()
+             ExternalProject_Use_Cache(bzip2_ep ${BZIP2_CACHE_PACKAGE_PATH} ${CMAKE_CURRENT_BINARY_DIR})   
+        endif()
+    else()
+        externalproject_add(bzip2_ep
+                                ${EP_LOG_OPTIONS}
+                                CONFIGURE_COMMAND
+                                ""
+                                BUILD_IN_SOURCE
+                                1
+                                BUILD_COMMAND
+                                ${MAKE}
+                                ${MAKE_BUILD_ARGS}
+                                CFLAGS=${EP_C_FLAGS}
+                                INSTALL_COMMAND
+                                ${MAKE}
+                                install
+                                PREFIX=${BZIP2_PREFIX}
+                                CFLAGS=${EP_C_FLAGS}
+                                INSTALL_DIR
+                                ${BZIP2_PREFIX}
+                                URL
+                                ${BZIP2_SOURCE_URL}
+                                BUILD_BYPRODUCTS
+                                "${BZIP2_STATIC_LIB}")    
+        file(MAKE_DIRECTORY "${BZIP2_INCLUDE_DIR}")
+    endif()    
+        
     add_library(bzip2 STATIC IMPORTED)
     set_target_properties(
             bzip2
@@ -1753,3 +1807,4 @@ if(MILVUS_WITH_GPERFTOOLS)
     include_directories(SYSTEM ${GPERFTOOLS_INCLUDE_DIR})
     link_directories(SYSTEM ${GPERFTOOLS_PREFIX}/lib)
 endif()
+