From d66d48c6b60c4cfe69b1ac1fc25002e045ee346b Mon Sep 17 00:00:00 2001 From: quicksilver Date: Tue, 27 Oct 2020 15:51:16 +0800 Subject: [PATCH] Enable UnitTest Signed-off-by: quicksilver --- .env | 3 + .github/workflows/main.yaml | 72 ++++++++------ .../cpu/ubuntu18.04/Dockerfile | 2 +- build/docker/{build_env => env}/entrypoint.sh | 0 deployments/docker/docker-compose.yml | 59 ++++++++++++ docker-compose.yml | 85 ++++++----------- docs/developer_guides/developer_guides.md | 19 ++-- internal/core/build.sh | 11 ++- internal/core/cmake/FindGTest.cmake | 58 ------------ internal/core/src/CMakeLists.txt | 44 --------- .../index/cmake/ThirdPartyPackagesCore.cmake | 94 ------------------- internal/core/src/main.cpp | 50 ---------- internal/core/thirdparty/CMakeLists.txt | 6 ++ internal/core/thirdparty/gtest/CMakeLists.txt | 64 +++++++++++++ .../core/thirdparty/protobuf/CMakeLists.txt | 1 + internal/core/unittest/CMakeLists.txt | 5 +- internal/proxy/manipulation_req.go | 32 +++---- internal/proxy/server.go | 9 +- scripts/core_build.sh | 2 +- 19 files changed, 242 insertions(+), 374 deletions(-) create mode 100644 .env rename build/docker/{build_env => env}/cpu/ubuntu18.04/Dockerfile (98%) rename build/docker/{build_env => env}/entrypoint.sh (100%) create mode 100644 deployments/docker/docker-compose.yml delete mode 100644 internal/core/cmake/FindGTest.cmake delete mode 100644 internal/core/src/main.cpp create mode 100644 internal/core/thirdparty/gtest/CMakeLists.txt diff --git a/.env b/.env new file mode 100644 index 0000000000..1c52e916c6 --- /dev/null +++ b/.env @@ -0,0 +1,3 @@ +REPO=milvusdb/milvus-distributed-dev +ARCH=amd64 +UBUNTU=18.04 diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml index 272a1793ac..32458677de 100644 --- a/.github/workflows/main.yaml +++ b/.github/workflows/main.yaml @@ -9,7 +9,7 @@ on: - 'scripts/**' - 'internal/**' - 'cmd/**' - - '.github/workflows/main.yml' + - '.github/workflows/main.yaml' - docker-compose.yml - '!**.md' - '!**_test.go' @@ -19,48 +19,58 @@ on: - 'scripts/**' - 'internal/**' - 'cmd/**' - - '.github/workflows/main.yml' + - '.github/workflows/main.yaml' - docker-compose.yml - '!**.md' - '!**_test.go' jobs: ubuntu: - name: AMD64 ubuntu-18.04 - runs-on: ubuntu-18.04 + name: AMD64 Ubuntu ${{ matrix.ubuntu }} + runs-on: ubuntu-latest + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: + ubuntu: [18.04] + env: + UBUNTU: ${{ matrix.ubuntu }} steps: - name: Checkout uses: actions/checkout@v2 - - name: Install Dependency - run: | - ./scripts/install_deps.sh - go get github.com/golang/protobuf/protoc-gen-go@v1.3.2 - - name: Cache Core Thirdparty - id: cache-core - uses: actions/cache@v2 + - name: Check Dockerfile + uses: reviewdog/action-hadolint@v1 with: - path: | - ./internal/core/cmake_build - key: ${{ runner.os }}-core-thirdparty - - name: Build Cpp + github_token: ${{ secrets.GITHUB_TOKEN }} + reporter: github-pr-check # Default is github-pr-check + - name: Docker Pull + shell: bash run: | - ./scripts/core_build.sh -u - - name: Generat Proto GO File + docker-compose pull --ignore-pull-failures ubuntu + - name: Docker Build + shell: bash run: | - echo `pwd` - pwd_dir=`pwd` - export PATH=$PATH:$(go env GOPATH)/bin - export protoc=${pwd_dir}/internal/core/cmake_build/thirdparty/protobuf/protobuf-build/protoc - ./scripts/proto_gen_go.sh - - name: Build GO + docker-compose build ubuntu + docker rmi $(docker images | grep '' | awk '{print $3}') || exit 0 + - name: Cache Docker Volumes + uses: actions/cache@v1 + with: + path: .docker + key: ubuntu${{ matrix.ubuntu }}-${{ hashFiles('internal/core/**') }} + restore-keys: ubuntu${{ matrix.ubuntu }}- + - name: Docker Run run: | - go build -o ./cmd/writer/writer ./cmd/writer/writer.go - go build -o ./cmd/reader/reader ./cmd/reader/reader.go - go build -o ./cmd/master/master ./cmd/master/master.go - go build -o ./cmd/proxy/proxy ./cmd/proxy/proxy.go - - name: Docker Pull And Run + docker-compose run ubuntu + - name: Run UnitTest + shell: bash run: | - docker-compose up -d - - name: Run Unittest + cd ${GITHUB_WORKSPACE}/deployments/docker && docker-compose up -d + cd ${GITHUB_WORKSPACE}/scripts/ && ./run_unittest.sh + - name: Docker Push + if: success() && github.event_name == 'push' && github.repository == 'zilliztech/milvus-distributed' + continue-on-error: true + shell: bash run: | - ./scripts/run_unittest.sh + docker login -u ${{ secrets.DOCKERHUB_USER }} \ + -p ${{ secrets.DOCKERHUB_TOKEN }} + docker-compose push ubuntu diff --git a/build/docker/build_env/cpu/ubuntu18.04/Dockerfile b/build/docker/env/cpu/ubuntu18.04/Dockerfile similarity index 98% rename from build/docker/build_env/cpu/ubuntu18.04/Dockerfile rename to build/docker/env/cpu/ubuntu18.04/Dockerfile index 75f48f8b10..4182873dd5 100644 --- a/build/docker/build_env/cpu/ubuntu18.04/Dockerfile +++ b/build/docker/env/cpu/ubuntu18.04/Dockerfile @@ -42,7 +42,7 @@ RUN mkdir -p /usr/local/go && wget -qO- "https://golang.org/dl/go1.15.2.linux-am go get github.com/golang/protobuf/protoc-gen-go@v1.3.2 # Set permissions on /etc/passwd and /home to allow arbitrary users to write -COPY --chown=0:0 docker/build_env/entrypoint.sh / +COPY --chown=0:0 build/docker/env/entrypoint.sh / RUN mkdir -p /home/user && chgrp -R 0 /home && chmod -R g=u /etc/passwd /etc/group /home && chmod +x /entrypoint.sh ENV HOME=/home/user diff --git a/build/docker/build_env/entrypoint.sh b/build/docker/env/entrypoint.sh similarity index 100% rename from build/docker/build_env/entrypoint.sh rename to build/docker/env/entrypoint.sh diff --git a/deployments/docker/docker-compose.yml b/deployments/docker/docker-compose.yml new file mode 100644 index 0000000000..9ad68a5463 --- /dev/null +++ b/deployments/docker/docker-compose.yml @@ -0,0 +1,59 @@ +version: '3.5' + +services: + etcd: + image: quay.io/coreos/etcd:latest + command: etcd -listen-peer-urls=http://127.0.0.1:12380 -advertise-client-urls=http://127.0.0.1:12379 -listen-client-urls http://0.0.0.0:12379,http://0.0.0.0:14001 -initial-advertise-peer-urls=http://127.0.0.1:12380 --initial-cluster default=http://127.0.0.1:12380 + ports: + - "12379:12379" + - "12380:12380" + - "14001:14001" + + pulsar: + image: apachepulsar/pulsar:latest + command: bin/pulsar standalone + ports: + - "6650:6650" + - "18080:8080" + + pd0: + image: pingcap/pd:latest + network_mode: "host" + ports: + - "2379:2379" + - "2380:2380" + volumes: + - /tmp/config/pd.toml:/pd.toml:ro + - /tmp/data:/data + - /tmp/logs:/logs + - /etc/localtime:/etc/localtime:ro + command: + - --name=pd0 + - --client-urls=http://0.0.0.0:2379 + - --peer-urls=http://0.0.0.0:2380 + - --advertise-client-urls=http://127.0.0.1:2379 + - --advertise-peer-urls=http://127.0.0.1:2380 + - --initial-cluster=pd0=http://127.0.0.1:2380 + - --data-dir=/data/pd0 + - --log-file=/logs/pd0.log + restart: on-failure + + tikv0: + network_mode: "host" + image: pingcap/tikv:latest + ports: + - "20160:20160" + volumes: + - /tmp/config/tikv.toml:/tikv.toml:ro + - /tmp/data:/data + - /tmp/logs:/logs + - /etc/localtime:/etc/localtime:ro + command: + - --addr=0.0.0.0:20160 + - --advertise-addr=127.0.0.1:20160 + - --data-dir=/data/tikv0 + - --pd=127.0.0.1:2379 + - --log-file=/logs/tikv0.log + depends_on: + - "pd0" + restart: on-failure diff --git a/docker-compose.yml b/docker-compose.yml index 1e399d1758..ad59f18c54 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,60 +1,31 @@ version: '3.5' +x-ccache: &ccache + CCACHE_COMPILERCHECK: content + CCACHE_COMPRESS: 1 + CCACHE_COMPRESSLEVEL: 5 + CCACHE_MAXSIZE: 2G + CCACHE_DIR: /ccache + services: - etcd: - image: quay.io/coreos/etcd:latest - command: etcd -listen-peer-urls=http://127.0.0.1:12380 -advertise-client-urls=http://127.0.0.1:12379 -listen-client-urls http://0.0.0.0:12379,http://0.0.0.0:14001 -initial-advertise-peer-urls=http://127.0.0.1:12380 --initial-cluster default=http://127.0.0.1:12380 - ports: - - "12379:12379" - - "12380:12380" - - "14001:14001" - - pulsar: - image: apachepulsar/pulsar:latest - command: bin/pulsar standalone - ports: - - "6650:6650" - - "18080:8080" - - pd0: - image: pingcap/pd:latest - network_mode: "host" - ports: - - "2379:2379" - - "2380:2380" - volumes: - - /tmp/config/pd.toml:/pd.toml:ro - - /tmp/data:/data - - /tmp/logs:/logs - - /etc/localtime:/etc/localtime:ro - command: - - --name=pd0 - - --client-urls=http://0.0.0.0:2379 - - --peer-urls=http://0.0.0.0:2380 - - --advertise-client-urls=http://127.0.0.1:2379 - - --advertise-peer-urls=http://127.0.0.1:2380 - - --initial-cluster=pd0=http://127.0.0.1:2380 - - --data-dir=/data/pd0 - - --log-file=/logs/pd0.log - restart: on-failure - - tikv0: - network_mode: "host" - image: pingcap/tikv:latest - ports: - - "20160:20160" - volumes: - - /tmp/config/tikv.toml:/tikv.toml:ro - - /tmp/data:/data - - /tmp/logs:/logs - - /etc/localtime:/etc/localtime:ro - command: - - --addr=0.0.0.0:20160 - - --advertise-addr=127.0.0.1:20160 - - --data-dir=/data/tikv0 - - --pd=127.0.0.1:2379 - - --log-file=/logs/tikv0.log - depends_on: - - "pd0" - restart: on-failure - + ubuntu: + image: ${REPO}:${ARCH}-ubuntu${UBUNTU} + build: + context: . + dockerfile: build/docker/env/cpu/ubuntu${UBUNTU}/Dockerfile + cache_from: + - ${REPO}:${ARCH}-ubuntu${UBUNTU} + shm_size: 2G + environment: + <<: *ccache + volumes: &ubuntu-volumes + - .:/milvus-distributed:delegated + - ${DOCKER_VOLUME_DIRECTORY:-.docker}/${ARCH}-ubuntu${UBUNTU}-cache:/ccache:delegated + working_dir: "/milvus-distributed" + command: &ubuntu-command > + /bin/bash -c " + /milvus-distributed/scripts/core_build.sh -u && \ + go build -o /milvus-distributed/cmd/writer/writer /milvus-distributed/cmd/writer/writer.go && \ + go build -o /milvus-distributed/cmd/reader/reader /milvus-distributed/cmd/reader/reader.go && \ + go build -o /milvus-distributed/cmd/master/master /milvus-distributed/cmd/master/master.go && \ + go build -o /milvus-distributed/cmd/proxy/proxy /milvus-distributed/cmd/proxy/proxy.go" diff --git a/docs/developer_guides/developer_guides.md b/docs/developer_guides/developer_guides.md index df1bb3719e..6f6354dcea 100644 --- a/docs/developer_guides/developer_guides.md +++ b/docs/developer_guides/developer_guides.md @@ -442,17 +442,17 @@ type TsMsg interface { Ts() Timestamp } -type TsMsgMarshaler interface { - Marshal(input *TsMsg) ([]byte, Status) - Unmarshal(input []byte) (*TsMsg, Status) -} - type MsgPack struct { BeginTs Timestamp EndTs Timestamp Msgs []*TsMsg } +type TsMsgMarshaler interface { + Marshal(input *TsMsg) ([]byte, Status) + Unmarshal(input []byte) (*TsMsg, Status) +} + type MsgStream interface { SetMsgMarshaler(marshal *TsMsgMarshaler, unmarshal *TsMsgMarshaler) Produce(*MsgPack) Status @@ -461,14 +461,17 @@ type MsgStream interface { type PulsarMsgStream struct { client *pulsar.Client - produceChannels []string - consumeChannels []string - + msgHashFunc (*MsgPack) map[int32]*MsgPack // return a map from produceChannel idx to *MsgPack + producers []*pulsar.Producer + consumers []*pulsar.Consumer msgMarshaler *TsMsgMarshaler msgUnmarshaler *TsMsgMarshaler } +func (ms *PulsarMsgStream) SetProducerChannels(channels []string) +func (ms *PulsarMsgStream) SetConsumerChannels(channels []string) func (ms *PulsarMsgStream) SetMsgMarshaler(marshal *TsMsgMarshaler, unmarshal *TsMsgMarshaler) +func (ms *PulsarMsgStream) SetMsgHashFunc(XXX) func (ms *PulsarMsgStream) Produce(*MsgPack) Status func (ms *PulsarMsgStream) Consume() *MsgPack //return messages in one time tick diff --git a/internal/core/build.sh b/internal/core/build.sh index 3881a63790..2af4ff088a 100755 --- a/internal/core/build.sh +++ b/internal/core/build.sh @@ -5,10 +5,19 @@ if [[ ! ${jobs+1} ]]; then jobs=$(nproc) fi +SOURCE="${BASH_SOURCE[0]}" +while [ -h "$SOURCE" ]; do # resolve $SOURCE until the file is no longer a symlink + DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" # if $SOURCE was a relative symlink, we need to resolve it relative to the path where the symlink file was located +done +SCRIPTS_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + + BUILD_OUTPUT_DIR="cmake_build" BUILD_TYPE="Release" BUILD_UNITTEST="OFF" -INSTALL_PREFIX=$(pwd)/output +INSTALL_PREFIX=${SCRIPTS_DIR}/output MAKE_CLEAN="OFF" BUILD_COVERAGE="OFF" DB_PATH="/tmp/milvus" diff --git a/internal/core/cmake/FindGTest.cmake b/internal/core/cmake/FindGTest.cmake deleted file mode 100644 index 726e737e71..0000000000 --- a/internal/core/cmake/FindGTest.cmake +++ /dev/null @@ -1,58 +0,0 @@ -find_package(Threads REQUIRED) - -include(ExternalProject) -ExternalProject_Add( - googletest - URL http://ss2.fluorinedog.com/data/gtest_v1.10.x.zip - UPDATE_COMMAND "" - INSTALL_COMMAND "" - LOG_DOWNLOAD ON - LOG_CONFIGURE ON - LOG_BUILD ON) - -ExternalProject_Get_Property(googletest source_dir) -set(GTEST_INCLUDE_DIRS ${source_dir}/googletest/include) -set(GMOCK_INCLUDE_DIRS ${source_dir}/googlemock/include) - -# The cloning of the above repo doesn't happen until make, however if the dir doesn't -# exist, INTERFACE_INCLUDE_DIRECTORIES will throw an error. -# To make it work, we just create the directory now during config. -file(MAKE_DIRECTORY ${GTEST_INCLUDE_DIRS}) -file(MAKE_DIRECTORY ${GMOCK_INCLUDE_DIRS}) - -ExternalProject_Get_Property(googletest binary_dir) -set(GTEST_LIBRARY_PATH ${binary_dir}/lib/${CMAKE_FIND_LIBRARY_PREFIXES}gtest.a) -set(GTEST_LIBRARY gtest) -add_library(${GTEST_LIBRARY} UNKNOWN IMPORTED) -set_target_properties(${GTEST_LIBRARY} PROPERTIES - "IMPORTED_LOCATION" "${GTEST_LIBRARY_PATH}" - "IMPORTED_LINK_INTERFACE_LIBRARIES" "${CMAKE_THREAD_LIBS_INIT}" - "INTERFACE_INCLUDE_DIRECTORIES" "${GTEST_INCLUDE_DIRS}") -add_dependencies(${GTEST_LIBRARY} googletest) - -set(GTEST_MAIN_LIBRARY_PATH ${binary_dir}/lib/${CMAKE_FIND_LIBRARY_PREFIXES}gtest_main.a) -set(GTEST_MAIN_LIBRARY gtest_main) -add_library(${GTEST_MAIN_LIBRARY} UNKNOWN IMPORTED) -set_target_properties(${GTEST_MAIN_LIBRARY} PROPERTIES - "IMPORTED_LOCATION" "${GTEST_MAIN_LIBRARY_PATH}" - "IMPORTED_LINK_INTERFACE_LIBRARImS" "${CMAKE_THREAD_LIBS_INIT}" - "INTERFACE_INCLUDE_DIRECTORIES" "${GTEST_INCLUDE_DIRS}") -add_dependencies(${GTEST_MAIN_LIBRARY} googletest) - -# set(GMOCK_LIBRARY_PATH ${binary_dir}/googlemock/${CMAKE_FIND_LIBRARY_PREFIXES}gmock.a) -# set(GMOCK_LIBRARY gmock) -# add_library(${GMOCK_LIBRARY} UNKNOWN IMPORTED) -# set_target_properties(${GMOCK_LIBRARY} PROPERTIES -# "IMPORTED_LOCATION" "${GMOCK_LIBRARY_PATH}" -# "IMPORTED_LINK_INTERFACE_LIBRARIES" "${CMAKE_THREAD_LIBS_INIT}" -# "INTERFACE_INCLUDE_DIRECTORIES" "${GMOCK_INCLUDE_DIRS}") -# add_dependencies(${GMOCK_LIBRARY} googletest) - -# set(GMOCK_MAIN_LIBRARY_PATH ${binary_dir}/googlemock/${CMAKE_FIND_LIBRARY_PREFIXES}gmock_main.a) -# set(GMOCK_MAIN_LIBRARY gmock_main) -# add_library(${GMOCK_MAIN_LIBRARY} UNKNOWN IMPORTED) -# set_target_properties(${GMOCK_MAIN_LIBRARY} PROPERTIES -# "IMPORTED_LOCATION" "${GMOCK_MAIN_LIBRARY_PATH}" -# "IMPORTED_LINK_INTERFACE_LIBRARIES" "${CMAKE_THREAD_LIBS_INIT}" -# "INTERFACE_INCLUDE_DIRECTORIES" "${GMOCK_INCLUDE_DIRS}") -# add_dependencies(${GMOCK_MAIN_LIBRARY} ${GTEST_LIBRARY}) \ No newline at end of file diff --git a/internal/core/src/CMakeLists.txt b/internal/core/src/CMakeLists.txt index 38fc44234a..bf5c61eb7a 100644 --- a/internal/core/src/CMakeLists.txt +++ b/internal/core/src/CMakeLists.txt @@ -29,47 +29,3 @@ add_subdirectory( log) add_subdirectory( dog_segment) add_subdirectory( cache ) add_subdirectory( query ) -# add_subdirectory( db ) # target milvus_engine -# add_subdirectory( server ) - -# set(link_lib -# milvus_engine -# # dog_segment -# #query -# utils -# curl -# ) - - -# set( BOOST_LIB libboost_system.a -# libboost_filesystem.a -# libboost_serialization.a -# ) - -# set( THIRD_PARTY_LIBS yaml-cpp -# ) - - -# target_link_libraries( server -# PUBLIC ${link_lib} -# ${THIRD_PARTY_LIBS} -# ${BOOST_LIB} -# ) - -# # **************************** Get&Print Include Directories **************************** -# get_property( dirs DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR} PROPERTY INCLUDE_DIRECTORIES ) - -# foreach ( dir ${dirs} ) -# message( STATUS "Current Include DIRS: ") -# endforeach () - -# set( SERVER_LIBS server ) - - -# add_executable( milvus_server ${CMAKE_CURRENT_SOURCE_DIR}/main.cpp -# ) -# #target_include_directories(db PUBLIC ${PROJECT_BINARY_DIR}/thirdparty/pulsar-client-cpp/pulsar-client-cpp-src/pulsar-client-cpp/include) - - -# target_link_libraries( milvus_server PRIVATE ${SERVER_LIBS} ) -# install( TARGETS milvus_server DESTINATION bin ) diff --git a/internal/core/src/index/cmake/ThirdPartyPackagesCore.cmake b/internal/core/src/index/cmake/ThirdPartyPackagesCore.cmake index 80c9763e66..72ad993918 100644 --- a/internal/core/src/index/cmake/ThirdPartyPackagesCore.cmake +++ b/internal/core/src/index/cmake/ThirdPartyPackagesCore.cmake @@ -14,7 +14,6 @@ set(KNOWHERE_THIRDPARTY_DEPENDENCIES Arrow FAISS - GTest OpenBLAS MKL ) @@ -31,8 +30,6 @@ endforeach () macro(build_dependency DEPENDENCY_NAME) if ("${DEPENDENCY_NAME}" STREQUAL "Arrow") build_arrow() - elseif ("${DEPENDENCY_NAME}" STREQUAL "GTest") - find_package(GTest REQUIRED) elseif ("${DEPENDENCY_NAME}" STREQUAL "OpenBLAS") build_openblas() elseif ("${DEPENDENCY_NAME}" STREQUAL "FAISS") @@ -215,12 +212,6 @@ else () ) endif () -# if (DEFINED ENV{KNOWHERE_GTEST_URL}) -# set(GTEST_SOURCE_URL "$ENV{KNOWHERE_GTEST_URL}") -# else () -# set(GTEST_SOURCE_URL -# "https://github.com/google/googletest/archive/release-${GTEST_VERSION}.tar.gz") -# endif () if (DEFINED ENV{KNOWHERE_OPENBLAS_URL}) set(OPENBLAS_SOURCE_URL "$ENV{KNOWHERE_OPENBLAS_URL}") @@ -383,91 +374,6 @@ if (KNOWHERE_WITH_OPENBLAS) link_directories(SYSTEM "${OpenBLAS_LIB_DIR}") endif() -# ---------------------------------------------------------------------- -# Google gtest - -# macro(build_gtest) -# message(STATUS "Building gtest-${GTEST_VERSION} from source") -# set(GTEST_VENDORED TRUE) -# set(GTEST_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}") -# -# if (APPLE) -# set(GTEST_CMAKE_CXX_FLAGS -# ${GTEST_CMAKE_CXX_FLAGS} -# -DGTEST_USE_OWN_TR1_TUPLE=1 -# -Wno-unused-value -# -Wno-ignored-attributes) -# endif () -# -# set(GTEST_PREFIX "${INDEX_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep") -# set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") -# set(GTEST_STATIC_LIB -# "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}") -# set(GTEST_MAIN_STATIC_LIB -# "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_STATIC_LIBRARY_SUFFIX}") -# -# set(GTEST_CMAKE_ARGS -# ${EP_COMMON_CMAKE_ARGS} -# "-DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX}" -# "-DCMAKE_INSTALL_LIBDIR=lib" -# -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS} -# -DCMAKE_BUILD_TYPE=Release) -# -# set(GMOCK_INCLUDE_DIR "${GTEST_PREFIX}/include") -# set(GMOCK_STATIC_LIB -# "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}" -# ) -# -# ExternalProject_Add(googletest_ep -# URL -# ${GTEST_SOURCE_URL} -# BUILD_COMMAND -# ${MAKE} -# ${MAKE_BUILD_ARGS} -# BUILD_BYPRODUCTS -# ${GTEST_STATIC_LIB} -# ${GTEST_MAIN_STATIC_LIB} -# ${GMOCK_STATIC_LIB} -# CMAKE_ARGS -# ${GTEST_CMAKE_ARGS} -# ${EP_LOG_OPTIONS}) -# -# # The include directory must exist before it is referenced by a target. -# file(MAKE_DIRECTORY "${GTEST_INCLUDE_DIR}") -# -# add_library(gtest STATIC IMPORTED) -# set_target_properties(gtest -# PROPERTIES IMPORTED_LOCATION "${GTEST_STATIC_LIB}" -# INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") -# -# add_library(gtest_main STATIC IMPORTED) -# set_target_properties(gtest_main -# PROPERTIES IMPORTED_LOCATION "${GTEST_MAIN_STATIC_LIB}" -# INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") -# -# add_library(gmock STATIC IMPORTED) -# set_target_properties(gmock -# PROPERTIES IMPORTED_LOCATION "${GMOCK_STATIC_LIB}" -# INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") -# -# add_dependencies(gtest googletest_ep) -# add_dependencies(gtest_main googletest_ep) -# add_dependencies(gmock googletest_ep) -# -# endmacro() - -## if (KNOWHERE_BUILD_TESTS AND NOT TARGET googletest_ep) -#if ( NOT TARGET gtest AND KNOWHERE_BUILD_TESTS ) -# resolve_dependency(GTest) -# -# if (NOT GTEST_VENDORED) -# endif () -# -# # TODO: Don't use global includes but rather target_include_directories -# get_target_property(GTEST_INCLUDE_DIR gtest INTERFACE_INCLUDE_DIRECTORIES) -# link_directories(SYSTEM "${GTEST_PREFIX}/lib") -# include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) -#endif () # ---------------------------------------------------------------------- # MKL diff --git a/internal/core/src/main.cpp b/internal/core/src/main.cpp deleted file mode 100644 index 4490a87c87..0000000000 --- a/internal/core/src/main.cpp +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright (C) 2019-2020 Zilliz. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software distributed under the License -// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -// or implied. See the License for the specific language governing permissions and limitations under the License. - -#include -#include -#include -#include -#include - -#include "config/ConfigMgr.h" -#include "easyloggingpp/easylogging++.h" -#include "utils/SignalHandler.h" -#include "utils/Status.h" - -INITIALIZE_EASYLOGGINGPP - -void -print_help(const std::string& app_name) { - std::cout << std::endl << "Usage: " << app_name << " [OPTIONS]" << std::endl; - std::cout << R"( - Options: - -h --help Print this help. - -c --conf_file filename Read configuration from the file. - -d --daemon Daemonize this application. - -p --pid_file filename PID file used by daemonized app. -)" << std::endl; -} - -void -print_banner() { - std::cout << std::endl; - std::cout << " __ _________ _ ____ ______ " << std::endl; - std::cout << " / |/ / _/ /| | / / / / / __/ " << std::endl; - std::cout << " / /|_/ // // /_| |/ / /_/ /\\ \\ " << std::endl; - std::cout << " /_/ /_/___/____/___/\\____/___/ " << std::endl; - std::cout << std::endl; -} - -int -main(int argc, char* argv[]) { - print_banner(); -} diff --git a/internal/core/thirdparty/CMakeLists.txt b/internal/core/thirdparty/CMakeLists.txt index 870cc0f850..e6421053fd 100644 --- a/internal/core/thirdparty/CMakeLists.txt +++ b/internal/core/thirdparty/CMakeLists.txt @@ -32,6 +32,12 @@ set( FETCHCONTENT_QUIET OFF ) set( THREADS_PREFER_PTHREAD_FLAG ON ) find_package( Threads REQUIRED ) +# ****************************** Thirdparty googletest *************************************** +if ( MILVUS_BUILD_TESTS ) + add_subdirectory( gtest ) +endif() + + # ****************************** Thirdparty yaml *************************************** if ( MILVUS_WITH_YAMLCPP ) add_subdirectory( yaml-cpp ) diff --git a/internal/core/thirdparty/gtest/CMakeLists.txt b/internal/core/thirdparty/gtest/CMakeLists.txt new file mode 100644 index 0000000000..0e6b65313e --- /dev/null +++ b/internal/core/thirdparty/gtest/CMakeLists.txt @@ -0,0 +1,64 @@ +#------------------------------------------------------------------------------- +# Copyright (C) 2019-2020 Zilliz. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under the License. +#------------------------------------------------------------------------------- + +if ( DEFINED ENV{MILVUS_GTEST_URL} ) + set( GTEST_SOURCE_URL "$ENV{MILVUS_GTEST_URL}" ) +else() + set( GTEST_SOURCE_URL "https://gitee.com/quicksilver/googletest/repository/archive/release-${GTEST_VERSION}.zip" ) +endif() + +message( STATUS "Building gtest-${GTEST_VERSION} from source" ) + +set( CMAKE_POLICY_DEFAULT_CMP0022 NEW ) # for googletest only + +FetchContent_Declare( + googletest + URL ${GTEST_SOURCE_URL} + URL_MD5 "f9137c5bc18b7d74027936f0f1bfa5c8" + SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}/googletest-src + BINARY_DIR ${CMAKE_CURRENT_BINARY_DIR}/googletest-build + DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH} ) + +FetchContent_GetProperties( googletest ) +if ( NOT googletest_POPULATED ) + + FetchContent_Populate( googletest ) + + # Adding the following targets: + # gtest, gtest_main, gmock, gmock_main + add_subdirectory( ${googletest_SOURCE_DIR} + ${googletest_BINARY_DIR} + EXCLUDE_FROM_ALL ) +endif() + +# **************************************************************** +# Create ALIAS Target +# **************************************************************** +# if (NOT TARGET GTest:gtest) +# add_library( GTest::gtest ALIAS gtest ) +# endif() +# if (NOT TARGET GTest:main) +# add_library( GTest::main ALIAS gtest_main ) +# endif() +# if (NOT TARGET GMock:gmock) +# target_link_libraries( gmock INTERFACE GTest::gtest ) +# add_library( GMock::gmock ALIAS gmock ) +# endif() +# if (NOT TARGET GMock:main) +# target_link_libraries( gmock_main INTERFACE GTest::gtest ) +# add_library( GMock::main ALIAS gmock_main ) +# endif() + + +get_property( var DIRECTORY "${CMAKE_CURRENT_BINARY_DIR}/googletest-src" PROPERTY COMPILE_OPTIONS ) +message( STATUS "gtest compile options: ${var}" ) diff --git a/internal/core/thirdparty/protobuf/CMakeLists.txt b/internal/core/thirdparty/protobuf/CMakeLists.txt index 93b1aac765..87e7f15ffa 100644 --- a/internal/core/thirdparty/protobuf/CMakeLists.txt +++ b/internal/core/thirdparty/protobuf/CMakeLists.txt @@ -31,6 +31,7 @@ FetchContent_Declare( ) +set( protobuf_BUILD_TESTS CACHE BOOL OFF FORCE ) if ( NOT protobuf_POPULATED ) FetchContent_Populate( protobuf ) diff --git a/internal/core/unittest/CMakeLists.txt b/internal/core/unittest/CMakeLists.txt index a9d87e0c42..8d5c07b144 100644 --- a/internal/core/unittest/CMakeLists.txt +++ b/internal/core/unittest/CMakeLists.txt @@ -1,8 +1,7 @@ enable_testing() -find_package(GTest REQUIRED) include_directories(${CMAKE_HOME_DIRECTORY}/src) -include_directories(>>>> ${CMAKE_HOME_DIRECTORY}/src/index/knowhere) +include_directories(${CMAKE_HOME_DIRECTORY}/src/index/knowhere) set(MILVUS_TEST_FILES test_naive.cpp test_dog_segment.cpp @@ -22,4 +21,4 @@ target_link_libraries(all_tests log pthread ) -install (TARGETS all_tests DESTINATION unittest) \ No newline at end of file +install (TARGETS all_tests DESTINATION unittest) diff --git a/internal/proxy/manipulation_req.go b/internal/proxy/manipulation_req.go index b2f9c84e26..8b4d75b69c 100644 --- a/internal/proxy/manipulation_req.go +++ b/internal/proxy/manipulation_req.go @@ -12,7 +12,7 @@ import ( ) type manipulationReq struct { - stats []commonpb.Status + commonpb.Status msgs []*pb.ManipulationReqMsg wg sync.WaitGroup proxy *proxyServer @@ -26,13 +26,14 @@ func (req *manipulationReq) Ts() (Timestamp, error) { return Timestamp(req.msgs[0].Timestamp), nil } func (req *manipulationReq) SetTs(ts Timestamp) { - for _, msg := range req.msgs { - msg.Timestamp = uint64(ts) + for _, mreq := range req.msgs { + mreq.Timestamp = uint64(ts) } } // BaseRequest interfaces func (req *manipulationReq) Type() pb.ReqType { + // TODO: return a invalid ReqType? if req.msgs == nil { return 0 } @@ -58,17 +59,11 @@ func (req *manipulationReq) Execute() commonpb.Status { func (req *manipulationReq) PostExecute() commonpb.Status { // send into pulsar req.wg.Add(1) - return commonpb.Status{ErrorCode: commonpb.ErrorCode_SUCCESS} + return req.Status } func (req *manipulationReq) WaitToFinish() commonpb.Status { // wait until send into pulsar req.wg.Wait() - - for _, stat := range req.stats{ - if stat.ErrorCode != commonpb.ErrorCode_SUCCESS{ - return stat - } - } // update timestamp if necessary ts, _ := req.Ts() req.proxy.reqSch.mTimestampMux.Lock() @@ -78,7 +73,7 @@ func (req *manipulationReq) WaitToFinish() commonpb.Status { // wait until send } else { log.Printf("there is some wrong with m_timestamp, it goes back, current = %d, previous = %d", ts, req.proxy.reqSch.mTimestamp) } - return req.stats[0] + return req.Status } func (s *proxyServer) restartManipulationRoutine(bufSize int) error { @@ -114,23 +109,22 @@ func (s *proxyServer) restartManipulationRoutine(bufSize int) error { ts, st := s.getTimestamp(1) if st.ErrorCode != commonpb.ErrorCode_SUCCESS { log.Printf("get time stamp failed, error code = %d, msg = %s", st.ErrorCode, st.Reason) - ip.stats[0] = st + ip.Status = st ip.wg.Done() break } ip.SetTs(ts[0]) wg := sync.WaitGroup{} - for i, mq := range ip.msgs { + for _, mq := range ip.msgs { mq := mq - i := i - wg.Add(1) go func() { + wg.Add(1) defer wg.Done() mb, err := proto.Marshal(mq) if err != nil { log.Printf("Marshal ManipulationReqMsg failed, error = %v", err) - ip.stats[i] = commonpb.Status{ + ip.Status = commonpb.Status{ ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR, Reason: fmt.Sprintf("Marshal ManipulationReqMsg failed, error=%v", err), } @@ -141,7 +135,7 @@ func (s *proxyServer) restartManipulationRoutine(bufSize int) error { case pb.ReqType_kInsert: if _, err := readers[mq.ChannelId].Send(s.ctx, &pulsar.ProducerMessage{Payload: mb}); err != nil { log.Printf("post into puslar failed, error = %v", err) - ip.stats[i] = commonpb.Status{ + ip.Status = commonpb.Status{ ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR, Reason: fmt.Sprintf("Post into puslar failed, error=%v", err.Error()), } @@ -150,10 +144,6 @@ func (s *proxyServer) restartManipulationRoutine(bufSize int) error { case pb.ReqType_kDeleteEntityByID: if _, err = deleter.Send(s.ctx, &pulsar.ProducerMessage{Payload: mb}); err != nil { log.Printf("post into pulsar filed, error = %v", err) - ip.stats[i] = commonpb.Status{ - ErrorCode: commonpb.ErrorCode_UNEXPECTED_ERROR, - Reason: fmt.Sprintf("Post into puslar failed, error=%v", err.Error()), - } return } default: diff --git a/internal/proxy/server.go b/internal/proxy/server.go index 38016452c4..3a2faf34c0 100644 --- a/internal/proxy/server.go +++ b/internal/proxy/server.go @@ -156,7 +156,6 @@ func (s *proxyServer) DeleteByID(ctx context.Context, req *pb.DeleteByIDParam) ( } if len(mReqMsg.PrimaryKeys) > 1 { mReq := &manipulationReq{ - stats: make([]commonpb.Status, 1), msgs: append([]*pb.ManipulationReqMsg{}, &mReqMsg), proxy: s, } @@ -223,10 +222,10 @@ func (s *proxyServer) Insert(ctx context.Context, req *servicepb.RowBatch) (*ser // TODO: alloc manipulation request id mReq := manipulationReq{ - stats: make([]commonpb.Status, len(msgMap)), - msgs: make([]*pb.ManipulationReqMsg, len(msgMap)), - wg: sync.WaitGroup{}, - proxy: s, + Status: commonpb.Status{}, + msgs: make([]*pb.ManipulationReqMsg, len(msgMap)), + wg: sync.WaitGroup{}, + proxy: s, } for _, v := range msgMap { mReq.msgs = append(mReq.msgs, v) diff --git a/scripts/core_build.sh b/scripts/core_build.sh index a7dc3f44fd..aef80771bd 100755 --- a/scripts/core_build.sh +++ b/scripts/core_build.sh @@ -19,7 +19,7 @@ CPP_BUILD_DIR="${CPP_SRC_DIR}/cmake_build" BUILD_OUTPUT_DIR=${CPP_BUILD_DIR} BUILD_TYPE="Release" BUILD_UNITTEST="OFF" -INSTALL_PREFIX="${CPP_SRC_DIR}/milvus" +INSTALL_PREFIX="${CPP_SRC_DIR}/output" MAKE_CLEAN="OFF" BUILD_COVERAGE="OFF" DB_PATH="/tmp/milvus"