From c7861c2dba05c89b7c6d8892d65fdedc72b04497 Mon Sep 17 00:00:00 2001 From: "cai.zhang" Date: Wed, 18 Nov 2020 10:07:05 +0800 Subject: [PATCH] Add the check of C++ code in the CI progress Signed-off-by: cai.zhang --- Makefile | 18 ++++++++-- build/docker/env/cpu/ubuntu18.04/Dockerfile | 2 +- docker-compose.yml | 2 +- internal/core/src/segcore/SegmentNaive.cpp | 38 +-------------------- internal/core/unittest/test_indexing.cpp | 38 +-------------------- scripts/check_proto_product.sh | 13 +++++++ scripts/gofmt.sh | 12 +++++++ scripts/run_clang_format.sh | 13 +++++++ 8 files changed, 57 insertions(+), 79 deletions(-) create mode 100755 scripts/check_proto_product.sh create mode 100755 scripts/gofmt.sh create mode 100755 scripts/run_clang_format.sh diff --git a/Makefile b/Makefile index 6f4447bc3d..2e14a01a93 100644 --- a/Makefile +++ b/Makefile @@ -27,10 +27,22 @@ get-check-deps: get-build-deps: @(env bash $(PWD)/scripts/install_deps.sh) +clang-format: + @echo "Running $@ check" + @(env bash ${PWD}/scripts/run_clang_format.sh internal/core) + +generated-proto-go:export protoc:=${PWD}/cmake_build/thirdparty/protobuf/protobuf-build/protoc +generated-proto-go: build-cpp + @(env bash $(PWD)/scripts/proto_gen_go.sh) + +check-proto-product: generated-proto-go + @(env bash $(PWD)/scripts/check_proto_product.sh) + fmt: @echo "Running $@ check" - @GO111MODULE=on gofmt -d cmd/ - @GO111MODULE=on gofmt -d internal/ + @GO111MODULE=on env bash $(PWD)/scripts/gofmt.sh cmd/ + @GO111MODULE=on env bash $(PWD)/scripts/gofmt.sh internal/ + @GO111MODULE=on env bash $(PWD)/scripts/gofmt.sh test/ #TODO: Check code specifications by golangci-lint lint: @@ -46,7 +58,7 @@ ruleguard: @${GOPATH}/bin/ruleguard -rules ruleguard.rules.go ./cmd/... @${GOPATH}/bin/ruleguard -rules ruleguard.rules.go ./test/... -verifiers: get-check-deps fmt lint ruleguard +verifiers: clang-format get-check-deps fmt lint ruleguard # Builds various components locally. build-go: diff --git a/build/docker/env/cpu/ubuntu18.04/Dockerfile b/build/docker/env/cpu/ubuntu18.04/Dockerfile index 69ad843701..5c1f53ea92 100644 --- a/build/docker/env/cpu/ubuntu18.04/Dockerfile +++ b/build/docker/env/cpu/ubuntu18.04/Dockerfile @@ -17,7 +17,7 @@ SHELL ["/bin/bash", "-o", "pipefail", "-c"] ENV DEBIAN_FRONTEND noninteractive -RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-certificates gnupg2 && \ +RUN apt-get update && apt-get install -y --no-install-recommends wget curl ca-certificates gnupg2 clang-format-10 && \ wget -qO- "https://cmake.org/files/v3.14/cmake-3.14.3-Linux-x86_64.tar.gz" | tar --strip-components=1 -xz -C /usr/local && \ apt-get update && apt-get install -y --no-install-recommends \ g++ gcc gfortran git make ccache libssl-dev zlib1g-dev libboost-regex-dev libboost-program-options-dev libboost-system-dev libboost-filesystem-dev libboost-serialization-dev python3-dev libboost-python-dev libcurl4-openssl-dev libtbb-dev clang-format-7 clang-tidy-7 lcov && \ diff --git a/docker-compose.yml b/docker-compose.yml index e2726d9699..444443abd7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -25,7 +25,7 @@ services: working_dir: "/go/src/github.com/zilliztech/milvus-distributed" command: &ubuntu-command > /bin/bash -c " - make build-cpp && make verifiers && make unittest" + make check-proto-product && make verifiers && make unittest" gdbserver: image: ${REPO}:${ARCH}-ubuntu${UBUNTU}-${DATE_VERSION} diff --git a/internal/core/src/segcore/SegmentNaive.cpp b/internal/core/src/segcore/SegmentNaive.cpp index f64c10247a..6e79976f00 100644 --- a/internal/core/src/segcore/SegmentNaive.cpp +++ b/internal/core/src/segcore/SegmentNaive.cpp @@ -8,6 +8,7 @@ #include #include #include +#include "segcore/Reduce.h" namespace milvus::segcore { @@ -294,43 +295,6 @@ SegmentNaive::QueryImpl(query::QueryDeprecatedPtr query_info, Timestamp timestam return Status::OK(); } -static void -merge_into(int64_t queries, - int64_t topk, - float* distances, - int64_t* uids, - const float* new_distances, - const int64_t* new_uids) { - for (int64_t qn = 0; qn < queries; ++qn) { - auto base = qn * topk; - auto src2_dis = distances + base; - auto src2_uids = uids + base; - - auto src1_dis = new_distances + base; - auto src1_uids = new_uids + base; - - std::vector buf_dis(topk); - std::vector buf_uids(topk); - - auto it1 = 0; - auto it2 = 0; - - for (auto buf = 0; buf < topk; ++buf) { - if (src1_dis[it1] <= src2_dis[it2]) { - buf_dis[buf] = src1_dis[it1]; - buf_uids[buf] = src1_uids[it1]; - ++it1; - } else { - buf_dis[buf] = src2_dis[it2]; - buf_uids[buf] = src2_uids[it2]; - ++it2; - } - } - std::copy_n(buf_dis.data(), topk, src2_dis); - std::copy_n(buf_uids.data(), topk, src2_uids); - } -} - Status SegmentNaive::QueryBruteForceImpl(query::QueryDeprecatedPtr query_info, Timestamp timestamp, QueryResult& results) { auto ins_barrier = get_barrier(record_, timestamp); diff --git a/internal/core/unittest/test_indexing.cpp b/internal/core/unittest/test_indexing.cpp index 05e4f987f7..ed01cc5a8c 100644 --- a/internal/core/unittest/test_indexing.cpp +++ b/internal/core/unittest/test_indexing.cpp @@ -20,6 +20,7 @@ #include #include #include "test_utils/Timer.h" +#include "segcore/Reduce.h" using std::cin; using std::cout; @@ -53,43 +54,6 @@ generate_data(int N) { } } // namespace -void -merge_into(int64_t queries, - int64_t topk, - float* distances, - int64_t* uids, - const float* new_distances, - const int64_t* new_uids) { - for (int64_t qn = 0; qn < queries; ++qn) { - auto base = qn * topk; - auto src2_dis = distances + base; - auto src2_uids = uids + base; - - auto src1_dis = new_distances + base; - auto src1_uids = new_uids + base; - - std::vector buf_dis(topk); - std::vector buf_uids(topk); - - auto it1 = 0; - auto it2 = 0; - - for (auto buf = 0; buf < topk; ++buf) { - if (src1_dis[it1] <= src2_dis[it2]) { - buf_dis[buf] = src1_dis[it1]; - buf_uids[buf] = src1_uids[it1]; - ++it1; - } else { - buf_dis[buf] = src2_dis[it2]; - buf_uids[buf] = src2_uids[it2]; - ++it2; - } - } - std::copy_n(buf_dis.data(), topk, src2_dis); - std::copy_n(buf_uids.data(), topk, src2_uids); - } -} - TEST(Indexing, SmartBruteForce) { // how to ? // I'd know diff --git a/scripts/check_proto_product.sh b/scripts/check_proto_product.sh new file mode 100755 index 0000000000..1ef60e64bf --- /dev/null +++ b/scripts/check_proto_product.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash + +SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + +cd ${SCRIPTS_DIR} + +GO_SRC_DIR="${SCRIPTS_DIR}/$1" +if test -z "$(git status | grep -E "*pb.go|*pb.cc|*pb.h")"; then + exit 0 +else + echo "The go file or cpp file generated by proto are not latest!" + exit 1 +fi diff --git a/scripts/gofmt.sh b/scripts/gofmt.sh new file mode 100755 index 0000000000..6364d3cf91 --- /dev/null +++ b/scripts/gofmt.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash + +SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + +GO_SRC_DIR="${SCRIPTS_DIR}/$1" +if test -z "$(gofmt -d $GO_SRC_DIR)"; then + exit 0 +else + gofmt -d $GO_SRC_DIR + echo "Please format your code by gofmt!" + exit 1 +fi diff --git a/scripts/run_clang_format.sh b/scripts/run_clang_format.sh new file mode 100755 index 0000000000..6c9d88a837 --- /dev/null +++ b/scripts/run_clang_format.sh @@ -0,0 +1,13 @@ +if [ -z $1 ]; then + echo "usage: $0 " + exit -1 +fi +CorePath=$1 + +formatThis() { + find "$1" | grep -E "(*\.cpp|*\.h|*\.cc)$" | grep -v "gen_tools/templates" | grep -v "/thirdparty" | grep -v "\.pb\." | xargs clang-format-10 -i +} + +formatThis "${CorePath}/src" +formatThis "${CorePath}/unittest" +