From 8348415d41613e9c38f86a696e2fb751a106e984 Mon Sep 17 00:00:00 2001 From: jinhai Date: Mon, 3 Jun 2019 11:52:53 +0800 Subject: [PATCH 1/5] Changelogs updated Former-commit-id: 220c49126526199c7f171a3435632b793fdf66d9 --- cpp/CHANGELOG.md | 26 +++++++++++++++----------- cpp/cmake/ThirdPartyPackages.cmake | 2 +- cpp/thirdparty/versions.txt | 4 ++-- 3 files changed, 18 insertions(+), 14 deletions(-) diff --git a/cpp/CHANGELOG.md b/cpp/CHANGELOG.md index 40563292bb..faa9c9cf84 100644 --- a/cpp/CHANGELOG.md +++ b/cpp/CHANGELOG.md @@ -2,30 +2,34 @@ Please mark all change in change log and use the ticket from JIRA. -## [Unreleased] +# MegaSearch 0.2.0 (2019-05-31) -### Bug +## Bug -### Improvement +- MS-32 - Fix thrift error +- MS-34 - Fix prometheus-cpp thirdparty -### New Feature +## Improvement + +- MS-20 - Clean Code Part 1 + +## New Feature - MS-5 - Implement Auto Archive Feature -- MS-16 - Implement metrics without prometheus - MS-6 - Implement SDK interface part 1 +- MS-16 - Implement metrics without prometheus - MS-21 - Implement SDK interface part 2 - MS-26 - cmake. Add thirdparty packages - MS-31 - cmake: add prometheus - MS-33 - cmake: add -j4 to make third party packages build faster - MS-27 - support gpu config and disable license build config in cmake - MS-47 - Add query vps metrics +- MS-37 - Add query, cache usage, disk write speed and file data size metrics +- MS-30 - Use faiss v1.5.2 +- MS-54 - cmake: Change Thirft third party URL to github.com -### Task +## Task - MS-1 - Add CHANGELOG.md - MS-4 - Refactor the vecwise_engine code structure -- MS-20 - Clean Code Part 1 -- MS-30 - Use faiss v1.5.2 -- MS-32 - Fix thrift error -- MS-34 - Fix prometheus-cpp thirdparty -- MS-37 - Add query, cache usage, disk write speed and file data size metrics + diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 8167879376..0a83d11446 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -288,7 +288,7 @@ if(DEFINED ENV{MEGASEARCH_THRIFT_URL}) set(THRIFT_SOURCE_URL "$ENV{MEGASEARCH_THRIFT_URL}") else() set(THRIFT_SOURCE_URL - "http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz") + "https://github.com/apache/thrift/archive/${THRIFT_VERSION}.tar.gz") endif() if(DEFINED ENV{MEGASEARCH_YAMLCPP_URL}) diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 6357f5dfa9..168b45e738 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -36,7 +36,7 @@ ROCKSDB_VERSION=v6.0.2 SNAPPY_VERSION=1.1.7 SQLITE_VERSION=3280000 SQLITE_ORM_VERSION=master -THRIFT_VERSION=0.12.0 +THRIFT_VERSION=v0.12.0 YAMLCPP_VERSION=0.6.2 ZLIB_VERSION=v1.2.11 ZSTD_VERSION=v1.4.0 @@ -59,7 +59,7 @@ DEPENDENCIES=( "MEGASEARCH_SNAPPY_URL snappy-${SNAPPY_VERSION}.tar.gz https://github.com/google/snappy/archive/${SNAPPY_VERSION}.tar.gz" "MEGASEARCH_SQLITE_URL sqlite-autoconf-${SQLITE_VERSION}.tar.gz https://www.sqlite.org/2019/sqlite-autoconf-${SQLITE_VERSION}.tar.gz" "MEGASEARCH_SQLITE_ORM_URL sqlite_orm-${SQLITE_ORM_VERSION}.tar.gz https://github.com/fnc12/sqlite_orm/archive/${SQLITE_ORM_VERSION}.tar.gz" - "MEGASEARCH_THRIFT_URL thrift-${THRIFT_VERSION}.tar.gz http://archive.apache.org/dist/thrift/${THRIFT_VERSION}/thrift-${THRIFT_VERSION}.tar.gz" + "MEGASEARCH_THRIFT_URL thrift-${THRIFT_VERSION}.tar.gz https://github.com/apache/thrift/archive/${THRIFT_VERSION}.tar.gz" "MEGASEARCH_YAMLCPP_URL yaml-cpp-${YAMLCPP_VERSION}.tar.gz https://github.com/jbeder/yaml-cpp/archive/yaml-cpp-${YAMLCPP_VERSION}.tar.gz" "MEGASEARCH_ZLIB_URL zlib-${ZLIB_VERSION}.tar.gz https://github.com/madler/zlib/archive/${ZLIB_VERSION}.tar.gz" "MEGASEARCH_ZSTD_URL zstd-${ZSTD_VERSION}.tar.gz https://github.com/facebook/zstd/archive/${ZSTD_VERSION}.tar.gz" From 3edb147b43c1351f5b341042b3805654d870455c Mon Sep 17 00:00:00 2001 From: jinhai Date: Mon, 3 Jun 2019 21:12:41 +0800 Subject: [PATCH 2/5] Use master branch of faiss Former-commit-id: 65339e9f2ea8e8cb98fe2679498b5e8b33d1f875 --- cpp/thirdparty/versions.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index 168b45e738..ca32b3b2a9 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -26,7 +26,7 @@ BOOST_VERSION=1.70.0 BZIP2_VERSION=1.0.6 EASYLOGGINGPP_VERSION=v9.96.7 -FAISS_VERSION=v1.5.2 +FAISS_VERSION=master GTEST_VERSION=1.8.1 LAPACK_VERSION=v3.8.0 LZ4_VERSION=v1.9.1 From 89d22d5927834e62b8ab5e173bf1d2a204188a33 Mon Sep 17 00:00:00 2001 From: jinhai Date: Tue, 4 Jun 2019 10:12:10 +0800 Subject: [PATCH 3/5] Still use faiss-1.5.0 and change the URL to customized REPO Former-commit-id: e6e1588032030474e755360d806501e6960e9022 --- cpp/cmake/ThirdPartyPackages.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index 0a83d11446..c09ae760b4 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -220,7 +220,7 @@ endif() if(DEFINED ENV{MEGASEARCH_FAISS_URL}) set(FAISS_SOURCE_URL "$ENV{MEGASEARCH_FAISS_URL}") else() - set(FAISS_SOURCE_URL "https://github.com/facebookresearch/faiss/archive/${FAISS_VERSION}.tar.gz") + set(FAISS_SOURCE_URL "https://github.com/JinHai-CN/faiss/archive/${FAISS_VERSION}.tar.gz") endif() if (DEFINED ENV{MEGASEARCH_GTEST_URL}) From 5554cd0e07adf0d69c3577a82e13c87a575bd1f7 Mon Sep 17 00:00:00 2001 From: jinhai Date: Tue, 4 Jun 2019 11:35:53 +0800 Subject: [PATCH 4/5] Update CMake files Former-commit-id: a67bcb2311cba6bff73597e1e45944992916ce4f --- cpp/cmake/ThirdPartyPackages.cmake | 2 +- cpp/thirdparty/versions.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cpp/cmake/ThirdPartyPackages.cmake b/cpp/cmake/ThirdPartyPackages.cmake index c09ae760b4..f42c279a8d 100644 --- a/cpp/cmake/ThirdPartyPackages.cmake +++ b/cpp/cmake/ThirdPartyPackages.cmake @@ -705,7 +705,7 @@ macro(build_faiss) # ${MAKE} ${MAKE_BUILD_ARGS} BUILD_COMMAND ${MAKE} - ${MAKE_BUILD_ARGS} + ${MAKE_BUILD_ARGS} all BUILD_IN_SOURCE 1 # INSTALL_DIR diff --git a/cpp/thirdparty/versions.txt b/cpp/thirdparty/versions.txt index ca32b3b2a9..b2d076ee56 100644 --- a/cpp/thirdparty/versions.txt +++ b/cpp/thirdparty/versions.txt @@ -49,7 +49,7 @@ DEPENDENCIES=( "MEGASEARCH_BOOST_URL boost-${BOOST_VERSION}.tar.gz https://dl.bintray.com/boostorg/release/${BOOST_VERSION}/source/boost_${BOOST_VERSION//./_}.tar.gz" "MEGASEARCH_BZIP2_URL bzip2-${BZIP2_VERSION}.tar.gz https://fossies.org/linux/misc/bzip2-${BZIP2_VERSION}.tar.gz" "MEGASEARCH_EASYLOGGINGPP_URL easyloggingpp-${EASYLOGGINGPP_VERSION}.tar.gz https://github.com/zuhd-org/easyloggingpp/archive/${EASYLOGGINGPP_VERSION}.tar.gz" - "MEGASEARCH_FAISS_URL faiss-${FAISS_VERSION}.tar.gz https://github.com/facebookresearch/faiss/archive/${FAISS_VERSION}.tar.gz" + "MEGASEARCH_FAISS_URL faiss-${FAISS_VERSION}.tar.gz https://github.com/JinHai-CN/faiss/archive/${FAISS_VERSION}.tar.gz" "MEGASEARCH_GTEST_URL gtest-${GTEST_VERSION}.tar.gz https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz" "MEGASEARCH_LAPACK_URL lapack-${LAPACK_VERSION}.tar.gz https://github.com/Reference-LAPACK/lapack/archive/${LAPACK_VERSION}.tar.gz "MEGASEARCH_LZ4_URL lz4-${LZ4_VERSION}.tar.gz https://github.com/lz4/lz4/archive/${LZ4_VERSION}.tar.gz" From d40f75e6df65feebeda62c2cdd0a2711e55f635c Mon Sep 17 00:00:00 2001 From: jinhai Date: Tue, 4 Jun 2019 11:58:05 +0800 Subject: [PATCH 5/5] Update CMakeLists.txt: add libgpufaiss.a Former-commit-id: 75eab2b1a069b6e42d3410543d95b38776a62a43 --- cpp/src/CMakeLists.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/cpp/src/CMakeLists.txt b/cpp/src/CMakeLists.txt index 86afe997dc..e400ab538f 100644 --- a/cpp/src/CMakeLists.txt +++ b/cpp/src/CMakeLists.txt @@ -99,6 +99,7 @@ if (GPU_VERSION STREQUAL "ON") set(engine_libs pthread libfaiss.a + libgpufaiss.a libgomp.a libopenblas.a libgfortran.a