Merge remote-tracking branch 'source/branch-0.5.0' into branch-0.5.0

Former-commit-id: da73035804cac289129cb02dd6eb8f7458bb9b87

cpp/CHANGELOG.md

@@ -25,6 +25,7 @@ Please mark all change in change log and use the ticket from JIRA.
 - MS-561 - Add contributing guidelines, code of conduct and README docs
 - MS-567 - Add NOTICE.md
 - MS-569 - Complete the NOTICE.md
+- MS-575 - Add Clang-format & Clang-tidy & Cpplint
 
 # Milvus 0.4.0 (2019-09-12)
 

cpp/CMakeLists.txt

@@ -21,6 +21,8 @@
 cmake_minimum_required(VERSION 3.14)
 message(STATUS "Building using CMake version: ${CMAKE_VERSION}")
 
+set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
+
 MACRO (GET_CURRENT_TIME CURRENT_TIME)
     execute_process(COMMAND "date" +"%Y-%m-%d %H:%M.%S" OUTPUT_VARIABLE ${CURRENT_TIME})
 ENDMACRO (GET_CURRENT_TIME)
@@ -41,6 +43,10 @@ endif()
 set(MILVUS_VERSION "${GIT_BRANCH_NAME}")
 string(REGEX MATCH "[0-9]+\\.[0-9]+\\.[0-9]" MILVUS_VERSION "${MILVUS_VERSION}")
 
+set(CLANG_FORMAT_VERSION "6.0")
+find_package(ClangTools)
+set(BUILD_SUPPORT_DIR "${CMAKE_SOURCE_DIR}/build-support")
+
 if(CMAKE_BUILD_TYPE STREQUAL "Release")
     set(BUILD_TYPE "Release")
 else()
@@ -111,8 +117,6 @@ else()
     include_directories(${MYSQL_INCLUDE_DIR})
 endif()
 
-set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
-
 set(MILVUS_SOURCE_DIR ${PROJECT_SOURCE_DIR})
 set(MILVUS_BINARY_DIR ${PROJECT_BINARY_DIR})
 set(MILVUS_ENGINE_SRC ${PROJECT_SOURCE_DIR}/src)
@@ -152,3 +156,9 @@ install(FILES
         conf/log_config.conf
         DESTINATION
         conf)
+
+add_custom_target(format ${BUILD_SUPPORT_DIR}/run_clang_format.py
+  ${CLANG_FORMAT_BIN}
+  ${BUILD_SUPPORT_DIR}/clang_format_exclusions.txt
+  ${CMAKE_CURRENT_SOURCE_DIR}/src)
+

cpp/build-support/run_clang_format.py

@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import fnmatch
+import os
+import subprocess
+import sys
+
+if len(sys.argv) < 4:
+    sys.stderr.write("Usage: %s $CLANG_FORMAT_VERSION exclude_globs.txt "
+                     "$source_dir\n" %
+                     sys.argv[0])
+    sys.exit(1)
+
+CLANG_FORMAT = sys.argv[1]
+EXCLUDE_GLOBS_FILENAME = sys.argv[2]
+SOURCE_DIR = sys.argv[3]
+
+if len(sys.argv) > 4:
+    CHECK_FORMAT = int(sys.argv[4]) == 1
+else:
+    CHECK_FORMAT = False
+
+
+exclude_globs = [line.strip() for line in open(EXCLUDE_GLOBS_FILENAME, "r")]
+
+files_to_format = []
+matches = []
+for directory, subdirs, files in os.walk(SOURCE_DIR):
+    for name in files:
+        name = os.path.join(directory, name)
+        if not (name.endswith('.h') or name.endswith('.cpp') or name.endswith('.cuh') or name.endswith('.cu')):
+            continue
+
+        excluded = False
+        for g in exclude_globs:
+            if fnmatch.fnmatch(name, g):
+                excluded = True
+                break
+        if not excluded:
+            files_to_format.append(name)
+
+if CHECK_FORMAT:
+    output = subprocess.check_output([CLANG_FORMAT, '-output-replacements-xml']
+                                     + files_to_format,
+                                     stderr=subprocess.STDOUT).decode('utf8')
+
+    to_fix = []
+    for line in output.split('\n'):
+        if 'offset' in line:
+            to_fix.append(line)
+
+    if len(to_fix) > 0:
+        print("clang-format checks failed, run 'make format' to fix")
+        sys.exit(-1)
+else:
+    try:
+        cmd = [CLANG_FORMAT, '-i'] + files_to_format
+        subprocess.check_output(cmd, stderr=subprocess.STDOUT)
+    except Exception as e:
+        print(e)
+        print(' '.join(cmd))
+        raise
+

cpp/cmake/FindClangTools.cmake

@@ -0,0 +1,104 @@
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Tries to find the clang-tidy and clang-format modules
+#
+# Usage of this module as follows:
+#
+#  find_package(ClangTools)
+#
+# Variables used by this module, they can change the default behaviour and need
+# to be set before calling find_package:
+#
+#  ClangToolsBin_HOME -
+#   When set, this path is inspected instead of standard library binary locations
+#   to find clang-tidy and clang-format
+#
+# This module defines
+#  CLANG_TIDY_BIN, The  path to the clang tidy binary
+#  CLANG_TIDY_FOUND, Whether clang tidy was found
+#  CLANG_FORMAT_BIN, The path to the clang format binary
+#  CLANG_TIDY_FOUND, Whether clang format was found
+
+find_program(CLANG_TIDY_BIN
+  NAMES
+  clang-tidy-6.0
+  clang-tidy-5.0
+  clang-tidy-4.0
+  clang-tidy-3.9
+  clang-tidy-3.8
+  clang-tidy-3.7
+  clang-tidy-3.6
+  clang-tidy
+  PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
+        NO_DEFAULT_PATH
+)
+
+if ( "${CLANG_TIDY_BIN}" STREQUAL "CLANG_TIDY_BIN-NOTFOUND" )
+  set(CLANG_TIDY_FOUND 0)
+  message("clang-tidy not found")
+else()
+  set(CLANG_TIDY_FOUND 1)
+  message("clang-tidy found at ${CLANG_TIDY_BIN}")
+endif()
+
+if (CLANG_FORMAT_VERSION)
+    find_program(CLANG_FORMAT_BIN
+      NAMES clang-format-${CLANG_FORMAT_VERSION}
+      PATHS
+            ${ClangTools_PATH}
+            $ENV{CLANG_TOOLS_PATH}
+            /usr/local/bin /usr/bin
+            NO_DEFAULT_PATH
+    )
+
+    # If not found yet, search alternative locations
+    if (("${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND") AND APPLE)
+        # Homebrew ships older LLVM versions in /usr/local/opt/llvm@version/
+        STRING(REGEX REPLACE "^([0-9]+)\\.[0-9]+" "\\1" CLANG_FORMAT_MAJOR_VERSION "${CLANG_FORMAT_VERSION}")
+        STRING(REGEX REPLACE "^[0-9]+\\.([0-9]+)" "\\1" CLANG_FORMAT_MINOR_VERSION "${CLANG_FORMAT_VERSION}")
+        if ("${CLANG_FORMAT_MINOR_VERSION}" STREQUAL "0")
+            find_program(CLANG_FORMAT_BIN
+              NAMES clang-format
+              PATHS /usr/local/opt/llvm@${CLANG_FORMAT_MAJOR_VERSION}/bin
+                    NO_DEFAULT_PATH
+            )
+        else()
+            find_program(CLANG_FORMAT_BIN
+              NAMES clang-format
+              PATHS /usr/local/opt/llvm@${CLANG_FORMAT_VERSION}/bin
+                    NO_DEFAULT_PATH
+            )
+        endif()
+    endif()
+else()
+    find_program(CLANG_FORMAT_BIN
+      NAMES clang-format-4.0
+      clang-format-3.9
+      clang-format-3.8
+      clang-format-3.7
+      clang-format-3.6
+      clang-format
+      PATHS ${ClangTools_PATH} $ENV{CLANG_TOOLS_PATH} /usr/local/bin /usr/bin
+            NO_DEFAULT_PATH
+    )
+endif()
+
+if ( "${CLANG_FORMAT_BIN}" STREQUAL "CLANG_FORMAT_BIN-NOTFOUND" )
+  set(CLANG_FORMAT_FOUND 0)
+  message("clang-format not found")
+else()
+  set(CLANG_FORMAT_FOUND 1)
+  message("clang-format found at ${CLANG_FORMAT_BIN}")
+endif()
+

cpp/src/core/knowhere/knowhere/common/Log.h

@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+
+#pragma once
+
+#include "utils/easylogging++.h"
+
+namespace zilliz {
+namespace knowhere {
+#define KNOWHERE_DOMAIN_NAME "[KNOWHERE] "
+#define KNOWHERE_ERROR_TEXT "KNOWHERE Error:"
+
+#define KNOWHERE_LOG_TRACE LOG(TRACE) << KNOWHERE_DOMAIN_NAME
+#define KNOWHERE_LOG_DEBUG LOG(DEBUG) << KNOWHERE_DOMAIN_NAME
+#define KNOWHERE_LOG_INFO LOG(INFO) << KNOWHERE_DOMAIN_NAME
+#define KNOWHERE_LOG_WARNING LOG(WARNING) << KNOWHERE_DOMAIN_NAME
+#define KNOWHERE_LOG_ERROR LOG(ERROR) << KNOWHERE_DOMAIN_NAME
+#define KNOWHERE_LOG_FATAL LOG(FATAL) << KNOWHERE_DOMAIN_NAME
+} // namespace knowhere
+} // namespace zilliz

cpp/src/core/knowhere/knowhere/common/exception.cpp

@@ -19,6 +19,7 @@
 #include <cstdio>
 
 #include "exception.h"
+#include "Log.h"
 
 namespace zilliz {
 namespace knowhere {

cpp/src/core/test/CMakeLists.txt

@@ -22,6 +22,9 @@ set(basic_libs
         gomp gfortran pthread
         )
 
+set(util_srcs
+        ${MILVUS_ENGINE_SRC}/utils/easylogging++.cc
+        )
 
 #<IVF-TEST>
 set(ivf_srcs
@@ -36,7 +39,7 @@ set(ivf_srcs
         utils.cpp
         )
 if(NOT TARGET test_ivf)
-    add_executable(test_ivf test_ivf.cpp ${ivf_srcs})
+    add_executable(test_ivf test_ivf.cpp ${ivf_srcs} ${util_srcs})
 endif()
 target_link_libraries(test_ivf ${depend_libs} ${unittest_libs} ${basic_libs})
 
@@ -53,7 +56,7 @@ set(idmap_srcs
         utils.cpp
         )
 if(NOT TARGET test_idmap)
-    add_executable(test_idmap test_idmap.cpp ${idmap_srcs})
+    add_executable(test_idmap test_idmap.cpp ${idmap_srcs} ${util_srcs})
 endif()
 target_link_libraries(test_idmap ${depend_libs} ${unittest_libs} ${basic_libs})
 
@@ -70,7 +73,7 @@ set(kdt_srcs
         utils.cpp
         )
 if(NOT TARGET test_kdt)
-    add_executable(test_kdt test_kdt.cpp ${kdt_srcs})
+    add_executable(test_kdt test_kdt.cpp ${kdt_srcs} ${util_srcs})
 endif()
 target_link_libraries(test_kdt
         SPTAGLibStatic

cpp/src/core/test/test_nsg/CMakeLists.txt

@@ -34,6 +34,7 @@ if(NOT TARGET test_nsg)
             test_nsg.cpp
             ${interface_src}
             ${nsg_src}
+            ${util_srcs}
             )
 endif()
 

cpp/src/core/test/utils.cpp

@@ -18,6 +18,15 @@
 
 #include "utils.h"
 
+INITIALIZE_EASYLOGGINGPP
+
+void InitLog() {
+    el::Configurations defaultConf;
+    defaultConf.setToDefault();
+    defaultConf.set(el::Level::Debug,
+                    el::ConfigurationType::Format, "[%thread-%datetime-%level]: %msg (%fbase:%line)");
+    el::Loggers::reconfigureLogger("default", defaultConf);
+}
 
 void DataGen::Init_with_default() {
     Generate(dim, nb, nq);

cpp/src/core/test/utils.h

@@ -24,6 +24,7 @@
 #include <fstream>
 
 #include "knowhere/adapter/structure.h"
+#include "knowhere/common/Log.h"
 
 class DataGen  {
  protected:
@@ -65,6 +66,8 @@ extern void GenBase(const int64_t &dim,
                     float *xb,
                     int64_t *ids);
 
+extern void InitLog();
+
 zilliz::knowhere::DatasetPtr
 generate_dataset(int64_t nb, int64_t dim, float *xb, long *ids);