diff --git a/build/rpm/milvus.spec b/build/rpm/milvus.spec index eb5b9cd1f1..2520549c3b 100644 --- a/build/rpm/milvus.spec +++ b/build/rpm/milvus.spec @@ -38,9 +38,7 @@ install -m 755 bin/minio %{buildroot}/usr/bin/milvus-minio # lib install -m 755 lib/libknowhere.so %{buildroot}/lib64/milvus/libknowhere.so -install -m 755 lib/libmilvus_common.so %{buildroot}/lib64/milvus/libmilvus_common.so -install -m 755 lib/libmilvus_indexbuilder.so %{buildroot}/lib64/milvus/libmilvus_indexbuilder.so -install -m 755 lib/libmilvus_segcore.so %{buildroot}/lib64/milvus/libmilvus_segcore.so +install -m 755 lib/libmilvus_core.so %{buildroot}/lib64/milvus/libmilvus_core.so install -m 755 /usr/lib/libopenblas-r0.3.9.so %{buildroot}/lib64/milvus/libopenblas.so.0 install -m 755 lib/libngt.so.1.12.0 %{buildroot}/lib64/milvus/libngt.so.1 install -m 755 /usr/lib64/libgfortran.so.4.0.0 %{buildroot}/lib64/milvus/libgfortran.so.4 @@ -82,9 +80,7 @@ systemctl daemon-reload /usr/bin/milvus-minio /lib64/milvus/libknowhere.so -/lib64/milvus/libmilvus_common.so -/lib64/milvus/libmilvus_indexbuilder.so -/lib64/milvus/libmilvus_segcore.so +/lib64/milvus/libmilvus_core.so /lib64/milvus/libopenblas.so.0 /lib64/milvus/libngt.so.1 /lib64/milvus/libgfortran.so.4 diff --git a/internal/core/CMakeLists.txt b/internal/core/CMakeLists.txt index 171209ccf6..10b3c3e76a 100644 --- a/internal/core/CMakeLists.txt +++ b/internal/core/CMakeLists.txt @@ -324,4 +324,3 @@ install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/futures/ install(DIRECTORY ${CMAKE_BINARY_DIR}/lib/ DESTINATION ${CMAKE_INSTALL_FULL_LIBDIR} ) - diff --git a/internal/core/cmake/Utils.cmake b/internal/core/cmake/Utils.cmake index f057a48457..8facf7cccd 100644 --- a/internal/core/cmake/Utils.cmake +++ b/internal/core/cmake/Utils.cmake @@ -99,4 +99,14 @@ macro(create_library) add_library(${L_TARGET} ${L_SRCS}) target_link_libraries(${L_TARGET} PRIVATE ${L_LIBS}) target_compile_definitions(${L_TARGET} PRIVATE ${L_DEFS}) -endmacro() \ No newline at end of file +endmacro() + +macro(add_source_at_current_directory_recursively) + file(GLOB_RECURSE SOURCE_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc" "*.cpp" "*.c" "*.cxx") + message(STATUS "${CMAKE_CURRENT_SOURCE_DIR} add new source files at current directory recursively: ${SOURCE_FILES}") +endmacro() + +macro(add_source_at_current_directory) + file(GLOB SOURCE_FILES RELATIVE ${CMAKE_CURRENT_SOURCE_DIR} "*.cc" "*.cpp" "*.c" "*.cxx") + message(STATUS "${CMAKE_CURRENT_SOURCE_DIR} add new source files at current directory: ${SOURCE_FILES}") +endmacro() diff --git a/internal/core/src/CMakeLists.txt b/internal/core/src/CMakeLists.txt index 7cc3961fad..6c0e831ae1 100644 --- a/internal/core/src/CMakeLists.txt +++ b/internal/core/src/CMakeLists.txt @@ -20,11 +20,23 @@ else() project(core CXX C) endif() +option( EMBEDDED_MILVUS "Enable embedded Milvus" OFF ) +if ( EMBEDDED_MILVUS ) + add_compile_definitions( EMBEDDED_MILVUS ) +endif() + include_directories(${MILVUS_ENGINE_SRC}) include_directories(${MILVUS_THIRDPARTY_SRC}) +include_directories( + ${KNOWHERE_INCLUDE_DIR} + ${SIMDJSON_INCLUDE_DIR} + ${TANTIVY_INCLUDE_DIR} + ${CONAN_INCLUDE_DIRS} + ${MILVUS_STORAGE_INCLUDE_DIRS} + ${MILVUS_STORAGE_SRC_DIRS} +) add_subdirectory( pb ) -add_subdirectory( log ) add_subdirectory( config ) add_subdirectory( common ) add_subdirectory( monitor ) @@ -37,3 +49,42 @@ add_subdirectory( clustering ) add_subdirectory( exec ) add_subdirectory( bitset ) add_subdirectory( futures ) + +milvus_add_pkg_config("milvus_core") + +add_library(milvus_core SHARED + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ + $ +) + +set(LINK_TARGETS + boost_bitset_ext + simdjson + tantivy_binding + knowhere + ${OpenMP_CXX_FLAGS} + ${CONAN_LIBS}) + +if(USE_OPENDAL) + set(LINK_TARGETS ${LINK_TARGETS} opendal) +endif() + +if(DEFINED AZURE_BUILD_DIR) + set(LINK_TARGETS ${LINK_TARGETS} azure_blob_chunk_manager) +endif() + +target_link_libraries(milvus_core ${LINK_TARGETS}) + +install(TARGETS milvus_core DESTINATION "${CMAKE_INSTALL_LIBDIR}") diff --git a/internal/core/src/bitset/CMakeLists.txt b/internal/core/src/bitset/CMakeLists.txt index 8b2137ca25..3f7c6ae24d 100644 --- a/internal/core/src/bitset/CMakeLists.txt +++ b/internal/core/src/bitset/CMakeLists.txt @@ -13,6 +13,8 @@ set(BITSET_SRCS detail/platform/dynamic.cpp ) + + if (${CMAKE_SYSTEM_PROCESSOR} STREQUAL "x86_64") list(APPEND BITSET_SRCS detail/platform/x86/avx2-inst.cpp @@ -38,4 +40,4 @@ elseif (${CMAKE_SYSTEM_PROCESSOR} MATCHES "arm*") #set_source_files_properties(detail/platform/arm/sve-inst.cpp PROPERTIES COMPILE_FLAGS "-mcpu=neoverse-v1") endif() -add_library(milvus_bitset ${BITSET_SRCS}) +add_library(milvus_bitset OBJECT ${BITSET_SRCS}) diff --git a/internal/core/src/clustering/CMakeLists.txt b/internal/core/src/clustering/CMakeLists.txt index 40833d9ef2..7331281499 100644 --- a/internal/core/src/clustering/CMakeLists.txt +++ b/internal/core/src/clustering/CMakeLists.txt @@ -9,16 +9,5 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under the License - -set(CLUSTERING_FILES - analyze_c.cpp - KmeansClustering.cpp - ) - -milvus_add_pkg_config("milvus_clustering") -add_library(milvus_clustering SHARED ${CLUSTERING_FILES}) - -# link order matters -target_link_libraries(milvus_clustering milvus_index) - -install(TARGETS milvus_clustering DESTINATION "${CMAKE_INSTALL_LIBDIR}") +add_source_at_current_directory_recursively() +add_library(milvus_clustering OBJECT ${SOURCE_FILES}) diff --git a/internal/core/src/clustering/milvus_clustering.pc.in b/internal/core/src/clustering/milvus_clustering.pc.in deleted file mode 100644 index d1bbb3d3ba..0000000000 --- a/internal/core/src/clustering/milvus_clustering.pc.in +++ /dev/null @@ -1,9 +0,0 @@ -libdir=@CMAKE_INSTALL_FULL_LIBDIR@ -includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ - -Name: Milvus Clustering -Description: Clustering modules for Milvus -Version: @MILVUS_VERSION@ - -Libs: -L${libdir} -lmilvus_clustering -Cflags: -I${includedir} diff --git a/internal/core/src/common/CMakeLists.txt b/internal/core/src/common/CMakeLists.txt index 4330b43f80..e1239c4d49 100644 --- a/internal/core/src/common/CMakeLists.txt +++ b/internal/core/src/common/CMakeLists.txt @@ -9,35 +9,5 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under the License -milvus_add_pkg_config("milvus_common") - -set(COMMON_SRC - Schema.cpp - SystemProperty.cpp - Slice.cpp - binary_set_c.cpp - init_c.cpp - Common.cpp - RangeSearchHelper.cpp - Tracer.cpp - IndexMeta.cpp - EasyAssert.cpp - FieldData.cpp - RegexQuery.cpp - ) - -add_library(milvus_common SHARED ${COMMON_SRC}) - -target_link_libraries(milvus_common - milvus_bitset - milvus_config - milvus_log - milvus_proto - yaml-cpp - boost_bitset_ext - simdjson - ${CONAN_LIBS} - re2 - ) - -install(TARGETS milvus_common DESTINATION "${CMAKE_INSTALL_LIBDIR}") +add_source_at_current_directory_recursively() +add_library(milvus_common OBJECT ${SOURCE_FILES}) diff --git a/internal/core/src/common/milvus_common.pc.in b/internal/core/src/common/milvus_common.pc.in deleted file mode 100644 index d1ae3f504f..0000000000 --- a/internal/core/src/common/milvus_common.pc.in +++ /dev/null @@ -1,9 +0,0 @@ -libdir=@CMAKE_INSTALL_FULL_LIBDIR@ -includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ - -Name: Milvus Common -Description: Common modules for Milvus -Version: @MILVUS_VERSION@ - -Libs: -L${libdir} -lmilvus_common -Cflags: -I${includedir} diff --git a/internal/core/src/config/CMakeLists.txt b/internal/core/src/config/CMakeLists.txt index 36f3ccc6a7..0fed02c95d 100644 --- a/internal/core/src/config/CMakeLists.txt +++ b/internal/core/src/config/CMakeLists.txt @@ -14,14 +14,5 @@ # See the License for the specific language governing permissions and # limitations under the License. -option( EMBEDDED_MILVUS "Enable embedded Milvus" OFF ) - -if ( EMBEDDED_MILVUS ) - add_compile_definitions( EMBEDDED_MILVUS ) -endif() - -set(CONFIG_SRC ConfigKnowhere.cpp) - -add_library(milvus_config STATIC ${CONFIG_SRC}) - -target_link_libraries(milvus_config knowhere) +add_source_at_current_directory_recursively() +add_library(milvus_config OBJECT ${SOURCE_FILES}) \ No newline at end of file diff --git a/internal/core/src/exec/CMakeLists.txt b/internal/core/src/exec/CMakeLists.txt index 9b1ca330c7..53c599ef80 100644 --- a/internal/core/src/exec/CMakeLists.txt +++ b/internal/core/src/exec/CMakeLists.txt @@ -9,25 +9,5 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under the License -set(MILVUS_EXEC_SRCS - expression/Expr.cpp - expression/UnaryExpr.cpp - expression/ConjunctExpr.cpp - expression/LogicalUnaryExpr.cpp - expression/LogicalBinaryExpr.cpp - expression/TermExpr.cpp - expression/BinaryArithOpEvalRangeExpr.cpp - expression/BinaryRangeExpr.cpp - expression/AlwaysTrueExpr.cpp - expression/CompareExpr.cpp - expression/JsonContainsExpr.cpp - expression/ExistsExpr.cpp - operator/FilterBits.cpp - operator/Operator.cpp - Driver.cpp - Task.cpp - ) - -add_library(milvus_exec STATIC ${MILVUS_EXEC_SRCS}) - -target_link_libraries(milvus_exec milvus_common milvus-storage ${CONAN_LIBS}) +add_source_at_current_directory_recursively() +add_library(milvus_exec OBJECT ${SOURCE_FILES}) diff --git a/internal/core/src/futures/CMakeLists.txt b/internal/core/src/futures/CMakeLists.txt index 59d4bdd9f2..9b948ac70e 100644 --- a/internal/core/src/futures/CMakeLists.txt +++ b/internal/core/src/futures/CMakeLists.txt @@ -9,16 +9,5 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under the License -milvus_add_pkg_config("milvus_futures") - -set(FUTURES_SRC - Executor.cpp - future_c.cpp - future_test_case_c.cpp - ) - -add_library(milvus_futures SHARED ${FUTURES_SRC}) - -target_link_libraries(milvus_futures milvus_common) - -install(TARGETS milvus_futures DESTINATION "${CMAKE_INSTALL_LIBDIR}") +add_source_at_current_directory_recursively() +add_library(milvus_futures OBJECT ${SOURCE_FILES}) \ No newline at end of file diff --git a/internal/core/src/futures/milvus_futures.pc.in b/internal/core/src/futures/milvus_futures.pc.in deleted file mode 100644 index dc75e325e8..0000000000 --- a/internal/core/src/futures/milvus_futures.pc.in +++ /dev/null @@ -1,9 +0,0 @@ -libdir=@CMAKE_INSTALL_FULL_LIBDIR@ -includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ - -Name: Milvus Futures -Description: Futures modules for Milvus -Version: @MILVUS_VERSION@ - -Libs: -L${libdir} -lmilvus_futures -Cflags: -I${includedir} diff --git a/internal/core/src/index/CMakeLists.txt b/internal/core/src/index/CMakeLists.txt index 2a3787f1c5..831e082869 100644 --- a/internal/core/src/index/CMakeLists.txt +++ b/internal/core/src/index/CMakeLists.txt @@ -9,22 +9,5 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under the License -set(INDEX_FILES - StringIndexMarisa.cpp - Utils.cpp - VectorMemIndex.cpp - IndexFactory.cpp - VectorDiskIndex.cpp - ScalarIndex.cpp - ScalarIndexSort.cpp - SkipIndex.cpp - BitmapIndex.cpp - InvertedIndexTantivy.cpp - ) - -milvus_add_pkg_config("milvus_index") -add_library(milvus_index SHARED ${INDEX_FILES}) - -target_link_libraries(milvus_index milvus_storage milvus-storage tantivy_binding) - -install(TARGETS milvus_index DESTINATION "${CMAKE_INSTALL_LIBDIR}") +add_source_at_current_directory_recursively() +add_library(milvus_index OBJECT ${SOURCE_FILES}) diff --git a/internal/core/src/indexbuilder/CMakeLists.txt b/internal/core/src/indexbuilder/CMakeLists.txt index a6fdea6456..da97de8037 100644 --- a/internal/core/src/indexbuilder/CMakeLists.txt +++ b/internal/core/src/indexbuilder/CMakeLists.txt @@ -9,18 +9,5 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under the License - -set(INDEXBUILDER_FILES - VecIndexCreator.cpp - index_c.cpp - init_c.cpp - ScalarIndexCreator.cpp - ) - -milvus_add_pkg_config("milvus_indexbuilder") -add_library(milvus_indexbuilder SHARED ${INDEXBUILDER_FILES}) - -# link order matters -target_link_libraries(milvus_indexbuilder milvus_index) - -install(TARGETS milvus_indexbuilder DESTINATION "${CMAKE_INSTALL_LIBDIR}") +add_source_at_current_directory_recursively() +add_library(milvus_indexbuilder OBJECT ${SOURCE_FILES}) \ No newline at end of file diff --git a/internal/core/src/indexbuilder/milvus_indexbuilder.pc.in b/internal/core/src/indexbuilder/milvus_indexbuilder.pc.in deleted file mode 100644 index f155bf1ca8..0000000000 --- a/internal/core/src/indexbuilder/milvus_indexbuilder.pc.in +++ /dev/null @@ -1,9 +0,0 @@ -libdir=@CMAKE_INSTALL_FULL_LIBDIR@ -includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ - -Name: Milvus IndexBuilder -Description: IndexBuilder modules for Milvus -Version: @MILVUS_VERSION@ - -Libs: -L${libdir} -lmilvus_indexbuilder -Cflags: -I${includedir} diff --git a/internal/core/src/log/CMakeLists.txt b/internal/core/src/log/CMakeLists.txt deleted file mode 100644 index 4332e43e97..0000000000 --- a/internal/core/src/log/CMakeLists.txt +++ /dev/null @@ -1,21 +0,0 @@ -#------------------------------------------------------------------------------- -# Copyright (C) 2019-2020 Zilliz. All rights reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software distributed under the License -# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions and limitations under the License. -#------------------------------------------------------------------------------- -set(LOG_FILES ${MILVUS_ENGINE_SRC}/log/Log.cpp - ${MILVUS_ENGINE_SRC}/log/Log.h - #${MILVUS_THIRDPARTY_SRC}/easyloggingpp/easylogging++.cc - #${MILVUS_THIRDPARTY_SRC}/easyloggingpp/easylogging++.h - ) - -add_library(milvus_log STATIC ${LOG_FILES}) -set_target_properties(milvus_log PROPERTIES RULE_LAUNCH_COMPILE "") -set_target_properties(milvus_log PROPERTIES RULE_LAUNCH_LINK "") diff --git a/internal/core/src/log/Log.cpp b/internal/core/src/log/Log.cpp deleted file mode 100644 index 4ec482e676..0000000000 --- a/internal/core/src/log/Log.cpp +++ /dev/null @@ -1,131 +0,0 @@ -// Licensed to the LF AI & Data foundation under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "log/Log.h" - -/* - * INITIALIZE_EASYLOGGINGPP will create a global variable whose name is same to that already created in knowhere, - * which will lead a `double-free` bug when the program exits. - * For why this issue happened please refer to - * https://gcc-help.gcc.gnu.narkive.com/KZGaXRNr/global-variable-in-static-library-double-free-or-corruption-error. - */ -// INITIALIZE_EASYLOGGINGPP - -#ifdef WIN32 -#include -#endif -#include -#include -#include -#include -#include -#include - -// namespace milvus { - -std::string -LogOut(const char* pattern, ...) { - size_t len = strnlen(pattern, 1024) + 256; - auto str_p = std::make_unique(len); - memset(str_p.get(), 0, len); - - va_list vl; - va_start(vl, pattern); - vsnprintf(str_p.get(), len, pattern, vl); // NOLINT - va_end(vl); - - return {str_p.get()}; -} - -void -SetThreadName(const std::string_view name) { - // Note: the name cannot exceed 16 bytes -#ifdef __APPLE__ - pthread_setname_np(name.data()); -#elif defined(__linux__) || defined(__MINGW64__) - pthread_setname_np(pthread_self(), name.data()); -#else -#error "Unsupported SetThreadName"; -#endif -} - -std::string -GetThreadName() { - std::string thread_name = "unnamed"; - char name[16]; - size_t len = 16; - auto err = pthread_getname_np(pthread_self(), name, len); - if (not err) { - thread_name = name; - } - - return thread_name; -} - -int64_t -get_now_timestamp() { - auto now = std::chrono::system_clock::now().time_since_epoch(); - return std::chrono::duration_cast(now).count(); -} - -#ifndef WIN32 - -int64_t -get_thread_starttime() { -#ifdef __APPLE__ - uint64_t tid; - pthread_threadid_np(nullptr, &tid); -#elif __linux__ - int64_t tid = gettid(); -#else -#error "Unsupported SetThreadName"; -#endif - - int64_t pid = getpid(); - char filename[256]; - snprintf(filename, - sizeof(filename), - "/proc/%lld/task/%lld/stat", - (long long)pid, // NOLINT, TODO: How to solve this? - (long long)tid); // NOLINT - - int64_t val = 0; - char comm[16], state; - FILE* thread_stat = fopen(filename, "r"); - auto ret = fscanf( - thread_stat, "%lld %s %s ", (long long*)&val, comm, &state); // NOLINT - - for (auto i = 4; i < 23; i++) { - ret = fscanf(thread_stat, "%lld ", (long long*)&val); // NOLINT - if (i == 22) { - break; - } - } - fclose(thread_stat); - if (ret != 1) { - throw std::runtime_error("read " + std::string(filename) + " failed."); - } - return val / sysconf(_SC_CLK_TCK); -} - -#else - -#define WINDOWS_TICK 10000000 -#define SEC_TO_UNIX_EPOCH 11644473600LL - -#endif - -// } // namespace milvus diff --git a/internal/core/src/log/Log.h b/internal/core/src/log/Log.h index b3d7f5170c..38bcf59b3a 100644 --- a/internal/core/src/log/Log.h +++ b/internal/core/src/log/Log.h @@ -14,6 +14,11 @@ // See the License for the specific language governing permissions and // limitations under the License. +// !!! Bad project layout here, the implementation is on +// 'milvus-storage/milvus-storage-src/cpp/src/common/log.cpp' The original +// implementation is on 'internal/core/src/log/Log.cpp', but odr-violation here, +// So the file is removed + #pragma once #include diff --git a/internal/core/src/index/milvus_index.pc.in b/internal/core/src/milvus_core.pc.in similarity index 59% rename from internal/core/src/index/milvus_index.pc.in rename to internal/core/src/milvus_core.pc.in index ffc4559bdf..eac5fd6c11 100644 --- a/internal/core/src/index/milvus_index.pc.in +++ b/internal/core/src/milvus_core.pc.in @@ -1,9 +1,9 @@ libdir=@CMAKE_INSTALL_FULL_LIBDIR@ includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ -Name: Milvus index -Description: index modules for Milvus +Name: Milvus Core +Description: Core modules for Milvus Version: @MILVUS_VERSION@ -Libs: -L${libdir} -lmilvus_index +Libs: -L${libdir} -lmilvus_core Cflags: -I${includedir} diff --git a/internal/core/src/monitor/CMakeLists.txt b/internal/core/src/monitor/CMakeLists.txt index 435dad13d9..01a8d6d9ea 100644 --- a/internal/core/src/monitor/CMakeLists.txt +++ b/internal/core/src/monitor/CMakeLists.txt @@ -9,15 +9,5 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under the License -milvus_add_pkg_config("milvus_monitor") - -set(MONITOR_SRC - monitor_c.cpp - prometheus_client.cpp - ) - -add_library(milvus_monitor SHARED ${MONITOR_SRC}) - -target_link_libraries(milvus_monitor PUBLIC ${CONAN_LIBS}) - -install(TARGETS milvus_monitor DESTINATION "${CMAKE_INSTALL_LIBDIR}") \ No newline at end of file +add_source_at_current_directory_recursively() +add_library(milvus_monitor OBJECT ${SOURCE_FILES}) \ No newline at end of file diff --git a/internal/core/src/monitor/milvus_monitor.pc.in b/internal/core/src/monitor/milvus_monitor.pc.in deleted file mode 100644 index 64d1b137b0..0000000000 --- a/internal/core/src/monitor/milvus_monitor.pc.in +++ /dev/null @@ -1,8 +0,0 @@ -libdir=@CMAKE_INSTALL_FULL_LIBDIR@ -includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ - -Name: Milvus Monitor -Description: Monitor modules for Milvus -Version: @MILVUS_VERSION@ - -Libs: -L${libdir} -lmilvus_monitor diff --git a/internal/core/src/pb/CMakeLists.txt b/internal/core/src/pb/CMakeLists.txt index d49637702d..899eaa2f18 100644 --- a/internal/core/src/pb/CMakeLists.txt +++ b/internal/core/src/pb/CMakeLists.txt @@ -11,11 +11,5 @@ find_package(Protobuf REQUIRED) -file(GLOB_RECURSE milvus_proto_srcs - "${CMAKE_CURRENT_SOURCE_DIR}/*.cc") -add_library(milvus_proto STATIC - ${milvus_proto_srcs} -) -message(STATUS "milvus proto sources: " ${milvus_proto_srcs}) - -target_link_libraries( milvus_proto PUBLIC ${CONAN_LIBS} ) +add_source_at_current_directory_recursively() +add_library(milvus_pb OBJECT ${SOURCE_FILES}) diff --git a/internal/core/src/query/CMakeLists.txt b/internal/core/src/query/CMakeLists.txt index 1609db31f8..79dec16aea 100644 --- a/internal/core/src/query/CMakeLists.txt +++ b/internal/core/src/query/CMakeLists.txt @@ -9,26 +9,5 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under the License -set(MILVUS_QUERY_SRCS - generated/PlanNode.cpp - generated/Expr.cpp - visitors/ShowPlanNodeVisitor.cpp - visitors/ShowExprVisitor.cpp - visitors/ExecPlanNodeVisitor.cpp - visitors/ExecExprVisitor.cpp - visitors/VerifyPlanNodeVisitor.cpp - visitors/VerifyExprVisitor.cpp - visitors/ExtractInfoPlanNodeVisitor.cpp - visitors/ExtractInfoExprVisitor.cpp - Plan.cpp - SearchOnGrowing.cpp - SearchOnSealed.cpp - SearchOnIndex.cpp - SearchBruteForce.cpp - SubSearchResult.cpp - GroupByOperator.cpp - PlanProto.cpp - ) -add_library(milvus_query ${MILVUS_QUERY_SRCS}) - -target_link_libraries(milvus_query milvus_index milvus_bitset milvus_monitor) +add_source_at_current_directory_recursively() +add_library(milvus_query OBJECT ${SOURCE_FILES}) diff --git a/internal/core/src/segcore/CMakeLists.txt b/internal/core/src/segcore/CMakeLists.txt index 7d7944eda9..52254660b8 100644 --- a/internal/core/src/segcore/CMakeLists.txt +++ b/internal/core/src/segcore/CMakeLists.txt @@ -9,39 +9,5 @@ # is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express # or implied. See the License for the specific language governing permissions and limitations under the License -option( EMBEDDED_MILVUS "Enable embedded Milvus" OFF ) - -if ( EMBEDDED_MILVUS ) - add_compile_definitions( EMBEDDED_MILVUS ) -endif() - -milvus_add_pkg_config("milvus_segcore") - -set(SEGCORE_FILES - Collection.cpp - collection_c.cpp - segment_c.cpp - SegmentGrowingImpl.cpp - SegmentSealedImpl.cpp - FieldIndexing.cpp - Reduce.cpp - StreamReduce.cpp - metrics_c.cpp - plan_c.cpp - reduce_c.cpp - load_index_c.cpp - load_field_data_c.cpp - SegmentInterface.cpp - SegcoreConfig.cpp - IndexConfigGenerator.cpp - segcore_init_c.cpp - TimestampIndex.cpp - Utils.cpp - ConcurrentVector.cpp - ReduceUtils.cpp - check_vec_index_c.cpp) -add_library(milvus_segcore SHARED ${SEGCORE_FILES}) - -target_link_libraries(milvus_segcore milvus_query milvus_bitset milvus_exec ${OpenMP_CXX_FLAGS} milvus-storage milvus_futures) - -install(TARGETS milvus_segcore DESTINATION "${CMAKE_INSTALL_LIBDIR}") +add_source_at_current_directory_recursively() +add_library(milvus_segcore OBJECT ${SOURCE_FILES}) diff --git a/internal/core/src/segcore/milvus_segcore.pc.in b/internal/core/src/segcore/milvus_segcore.pc.in deleted file mode 100644 index 7b367dbb37..0000000000 --- a/internal/core/src/segcore/milvus_segcore.pc.in +++ /dev/null @@ -1,9 +0,0 @@ -libdir=@CMAKE_INSTALL_FULL_LIBDIR@ -includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ - -Name: Milvus Segcore -Description: Segcore modules for Milvus -Version: @MILVUS_VERSION@ - -Libs: -L${libdir} -lmilvus_segcore -Cflags: -I${includedir} diff --git a/internal/core/src/storage/CMakeLists.txt b/internal/core/src/storage/CMakeLists.txt index aa4b02f640..0d69386e08 100644 --- a/internal/core/src/storage/CMakeLists.txt +++ b/internal/core/src/storage/CMakeLists.txt @@ -14,74 +14,27 @@ # See the License for the specific language governing permissions and # limitations under the License. -option(EMBEDDED_MILVUS "Enable embedded Milvus" OFF) - -if(EMBEDDED_MILVUS) - add_compile_definitions(EMBEDDED_MILVUS) -endif() - -milvus_add_pkg_config("milvus_storage") - +add_source_at_current_directory() if (DEFINED AZURE_BUILD_DIR) add_definitions(-DAZURE_BUILD_DIR) include_directories(azure-blob-storage) include_directories("${AZURE_BUILD_DIR}/vcpkg_installed/${VCPKG_TARGET_TRIPLET}/include") - set(STORAGE_FILES - ${STORAGE_FILES} - AzureChunkManager.cpp - ) + set(SOURCE_FILES ${SOURCE_FILES} azure/AzureChunkManager.cpp) + add_library(azure_blob_chunk_manager SHARED IMPORTED) + set_target_properties(azure_blob_chunk_manager + PROPERTIES + IMPORTED_GLOBAL TRUE + IMPORTED_LOCATION "${AZURE_BUILD_DIR}/libblob-chunk-manager${CMAKE_SHARED_LIBRARY_SUFFIX}" + ) + get_target_property(AZURE_IMPORTED_LOCATION azure_blob_chunk_manager IMPORTED_LOCATION) + get_target_property(AZURE_INTERFACE_INCLUDE_DIRECTORIES azure_blob_chunk_manager INTERFACE_INCLUDE_DIRECTORIES) + message("AZURE_IMPORTED_LOCATION: ${AZURE_IMPORTED_LOCATION}") + message("AZURE_INTERFACE_INCLUDE_DIRECTORIES: ${AZURE_INTERFACE_INCLUDE_DIRECTORIES}") endif() -set(STORAGE_FILES - ${STORAGE_FILES} - PayloadStream.cpp - DataCodec.cpp - Util.cpp - PayloadReader.cpp - PayloadWriter.cpp - BinlogReader.cpp - IndexData.cpp - InsertData.cpp - Event.cpp - ThreadPool.cpp - storage_c.cpp - ChunkManager.cpp - MinioChunkManager.cpp - AliyunSTSClient.cpp - AliyunCredentialsProvider.cpp - MemFileManagerImpl.cpp - LocalChunkManager.cpp - DiskFileManagerImpl.cpp - ThreadPools.cpp - ChunkCache.cpp - TencentCloudCredentialsProvider.cpp - TencentCloudSTSClient.cpp - MmapChunkManager.cpp) - if(USE_OPENDAL) - list(APPEND STORAGE_FILES OpenDALChunkManager.cpp) + include_directories(${OPENDAL_INCLUDE_DIR}) + set(SOURCE_FILES ${SOURCE_FILES} opendal/OpenDALChunkManager.cpp) endif() -add_library(milvus_storage SHARED ${STORAGE_FILES}) - -if (DEFINED AZURE_BUILD_DIR) - target_link_libraries(milvus_storage PUBLIC - "-L${AZURE_BUILD_DIR} -lblob-chunk-manager" - blob-chunk-manager - milvus_common - milvus-storage - milvus_monitor - pthread - ${CONAN_LIBS} - ) -else () - target_link_libraries(milvus_storage PUBLIC - milvus_common - milvus-storage - milvus_monitor - pthread - ${CONAN_LIBS} - ) -endif() - -install(TARGETS milvus_storage DESTINATION "${CMAKE_INSTALL_LIBDIR}") +add_library(milvus_storage OBJECT ${SOURCE_FILES}) diff --git a/internal/core/src/storage/Util.cpp b/internal/core/src/storage/Util.cpp index 75973ed37b..1e3579666b 100644 --- a/internal/core/src/storage/Util.cpp +++ b/internal/core/src/storage/Util.cpp @@ -27,7 +27,7 @@ #include "common/FieldData.h" #include "common/FieldDataInterface.h" #ifdef AZURE_BUILD_DIR -#include "storage/AzureChunkManager.h" +#include "storage/azure/AzureChunkManager.h" #endif #include "storage/ChunkManager.h" #include "storage/DiskFileManagerImpl.h" @@ -36,7 +36,7 @@ #include "storage/MemFileManagerImpl.h" #include "storage/MinioChunkManager.h" #ifdef USE_OPENDAL -#include "storage/OpenDALChunkManager.h" +#include "storage/opendal/OpenDALChunkManager.h" #endif #include "storage/Types.h" #include "storage/Util.h" diff --git a/internal/core/src/storage/azure-blob-storage/CMakeLists.txt b/internal/core/src/storage/azure-blob-storage/CMakeLists.txt index 62b2e971c8..cf6ffa0f72 100644 --- a/internal/core/src/storage/azure-blob-storage/CMakeLists.txt +++ b/internal/core/src/storage/azure-blob-storage/CMakeLists.txt @@ -32,10 +32,10 @@ endif() find_package(azure-storage-blobs-cpp CONFIG REQUIRED) find_package(azure-identity-cpp CONFIG REQUIRED) -set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter -Wno-return-type -Wno-pedantic") +set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter -Wno-return-type -Wno-pedantic -fPIC") add_library(blob-chunk-manager SHARED AzureBlobChunkManager.cpp) target_link_libraries(blob-chunk-manager PUBLIC Azure::azure-identity Azure::azure-storage-blobs) - +# should be link directly into libmilvus_core in future. install(TARGETS blob-chunk-manager DESTINATION "${CMAKE_INSTALL_LIBDIR}") if ( BUILD_UNIT_TEST STREQUAL "ON" ) diff --git a/internal/core/src/storage/AzureChunkManager.cpp b/internal/core/src/storage/azure/AzureChunkManager.cpp similarity index 99% rename from internal/core/src/storage/AzureChunkManager.cpp rename to internal/core/src/storage/azure/AzureChunkManager.cpp index 99a29a8142..854504e28a 100644 --- a/internal/core/src/storage/AzureChunkManager.cpp +++ b/internal/core/src/storage/azure/AzureChunkManager.cpp @@ -21,7 +21,7 @@ #include "common/EasyAssert.h" #include "log/Log.h" #include "monitor/prometheus_client.h" -#include "storage/AzureChunkManager.h" +#include "storage/azure/AzureChunkManager.h" namespace milvus { namespace storage { diff --git a/internal/core/src/storage/AzureChunkManager.h b/internal/core/src/storage/azure/AzureChunkManager.h similarity index 100% rename from internal/core/src/storage/AzureChunkManager.h rename to internal/core/src/storage/azure/AzureChunkManager.h diff --git a/internal/core/src/storage/milvus_storage.pc.in b/internal/core/src/storage/milvus_storage.pc.in deleted file mode 100644 index 6ad79e91ac..0000000000 --- a/internal/core/src/storage/milvus_storage.pc.in +++ /dev/null @@ -1,10 +0,0 @@ - -libdir=@CMAKE_INSTALL_FULL_LIBDIR@ -includedir=@CMAKE_INSTALL_FULL_INCLUDEDIR@ - -Name: Milvus Storage -Description: Storage wrapper for Milvus -Version: @MILVUS_VERSION@ - -Libs: -L${libdir} -lmilvus_storage -Cflags: -I${includedir} diff --git a/internal/core/src/storage/OpenDALChunkManager.cpp b/internal/core/src/storage/opendal/OpenDALChunkManager.cpp similarity index 99% rename from internal/core/src/storage/OpenDALChunkManager.cpp rename to internal/core/src/storage/opendal/OpenDALChunkManager.cpp index 3affe3ae07..465c0a939a 100644 --- a/internal/core/src/storage/OpenDALChunkManager.cpp +++ b/internal/core/src/storage/opendal/OpenDALChunkManager.cpp @@ -22,7 +22,7 @@ #include "opendal.h" #include "common/EasyAssert.h" #include "storage/Util.h" -#include "storage/OpenDALChunkManager.h" +#include "storage/opendal/OpenDALChunkManager.h" namespace milvus::storage { diff --git a/internal/core/src/storage/OpenDALChunkManager.h b/internal/core/src/storage/opendal/OpenDALChunkManager.h similarity index 100% rename from internal/core/src/storage/OpenDALChunkManager.h rename to internal/core/src/storage/opendal/OpenDALChunkManager.h diff --git a/internal/core/thirdparty/knowhere/CMakeLists.txt b/internal/core/thirdparty/knowhere/CMakeLists.txt index 8be3a8e0f0..148c98429a 100644 --- a/internal/core/thirdparty/knowhere/CMakeLists.txt +++ b/internal/core/thirdparty/knowhere/CMakeLists.txt @@ -59,3 +59,5 @@ endif() # get prometheus COMPILE_OPTIONS get_property( var DIRECTORY "${knowhere_SOURCE_DIR}" PROPERTY COMPILE_OPTIONS ) message( STATUS "knowhere src compile options: ${var}" ) + +set( KNOWHERE_INCLUDE_DIR ${knowhere_SOURCE_DIR}/include CACHE INTERNAL "Path to knowhere include directory" ) diff --git a/internal/core/thirdparty/milvus-storage/CMakeLists.txt b/internal/core/thirdparty/milvus-storage/CMakeLists.txt index a67a7cae82..77b7f6ff36 100644 --- a/internal/core/thirdparty/milvus-storage/CMakeLists.txt +++ b/internal/core/thirdparty/milvus-storage/CMakeLists.txt @@ -46,3 +46,6 @@ FetchContent_MakeAvailable(milvus-storage) # get_property( var DIRECTORY "${milvus-storage_SOURCE_DIR}" PROPERTY COMPILE_OPTIONS ) message( STATUS "milvus-storage src compile options: ${var}" ) # unset(CMAKE_CXX_STANDARD) + +set( MILVUS_STORAGE_INCLUDE_DIRS ${milvus-storage_SOURCE_DIR}/cpp/include/milvus-storage CACHE INTERNAL "Path to milvus storage include directory" ) +set( MILVUS_STORAGE_SRC_DIRS ${milvus-storage_SOURCE_DIR}/cpp/src CACHE INTERNAL "Path to milvus storage src include directory" ) \ No newline at end of file diff --git a/internal/core/thirdparty/milvus-storage/milvus-storage_CMakeLists.txt b/internal/core/thirdparty/milvus-storage/milvus-storage_CMakeLists.txt index 135765c99e..2b3d09c718 100644 --- a/internal/core/thirdparty/milvus-storage/milvus-storage_CMakeLists.txt +++ b/internal/core/thirdparty/milvus-storage/milvus-storage_CMakeLists.txt @@ -21,12 +21,8 @@ find_package(AWSSDK REQUIRED) file(GLOB_RECURSE SRC_FILES src/*.cpp src/*.cc) message(STATUS "SRC_FILES: ${SRC_FILES}") -add_library(milvus-storage ${SRC_FILES}) +add_library(milvus-storage OBJECT ${SRC_FILES}) target_include_directories(milvus-storage PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/include/milvus-storage ${CMAKE_CURRENT_SOURCE_DIR}/src) -target_link_libraries(milvus-storage PUBLIC arrow::arrow Boost::boost protobuf::protobuf AWS::aws-sdk-cpp-core glog::glog) -if (USE_OPENDAL) - target_link_libraries(milvus-storage PUBLIC opendal) -endif() if (WITH_UT) enable_testing() diff --git a/internal/core/thirdparty/opendal/CMakeLists.txt b/internal/core/thirdparty/opendal/CMakeLists.txt index 25fd05e83c..a01f153621 100644 --- a/internal/core/thirdparty/opendal/CMakeLists.txt +++ b/internal/core/thirdparty/opendal/CMakeLists.txt @@ -14,16 +14,48 @@ # ---------------------------------------------------------------------- message(STATUS "Building (vendored) opendal from source") -set(OPENDAL_LIB_DIR "${CMAKE_INSTALL_PREFIX}/lib") -set(OPENDAL_INCLUDE_DIR "${CMAKE_INSTALL_PREFIX}/include") -set(OPENDAL_NAME "libopendal_c${CMAKE_STATIC_LIBRARY_SUFFIX}") +set(GIT_REPOSITORY "https://github.com/apache/opendal.git") +set(GIT_TAG "v0.43.0-rc.2") + +if (CMAKE_BUILD_TYPE STREQUAL "Debug") + set(CARGO_CMD cargo +1.73 build --verbose) + set(TARGET_DIR "debug") +else () + set(CARGO_CMD cargo +1.73 build --release --verbose) + set(TARGET_DIR "release") +endif () + +set(SOURCE_DIR "${CMAKE_CURRENT_BINARY_DIR}/src") + +FetchContent_Declare( + opendal + GIT_REPOSITORY ${GIT_REPOSITORY} + GIT_TAG ${GIT_TAG} + GIT_SHALLOW TRUE + SOURCE_DIR ${SOURCE_DIR} + DOWNLOAD_DIR ${THIRDPARTY_DOWNLOAD_PATH}) + +FetchContent_GetProperties(opendal) +if ( NOT opendal_POPULATED ) + FetchContent_Populate(opendal) +endif() + +set(OPENDAL_LIB_DIR "${SOURCE_DIR}/target/${TARGET_DIR}" CACHE INTERNAL "opendal lib dir") +set(OPENDAL_INCLUDE_DIR "${SOURCE_DIR}/bindings/c/include" CACHE INTERNAL "opendal include dir") +set(OPENDAL_LIB "libopendal_c${CMAKE_STATIC_LIBRARY_SUFFIX}" CACHE INTERNAL "opendal lib") + +add_custom_target(build_opendal + COMMAND ${CARGO_CMD} + WORKING_DIRECTORY ${SOURCE_DIR}/bindings/c +) add_library(opendal STATIC IMPORTED) +add_dependencies(opendal build_opendal) set_target_properties(opendal PROPERTIES IMPORTED_GLOBAL TRUE - IMPORTED_LOCATION "${OPENDAL_LIB_DIR}/${OPENDAL_NAME}" - INTERFACE_INCLUDE_DIRECTORIES "${CMAKE_INSTALL_PREFIX}/include") + IMPORTED_LOCATION "${OPENDAL_LIB_DIR}/${OPENDAL_LIB}" + INTERFACE_INCLUDE_DIRECTORIES "${OPENDAL_INCLUDE_DIR}") get_target_property(OPENDAL_IMPORTED_LOCATION opendal IMPORTED_LOCATION) get_target_property(OPENDAL_INTERFACE_INCLUDE_DIRECTORIES opendal INTERFACE_INCLUDE_DIRECTORIES) diff --git a/internal/core/thirdparty/simdjson/CMakeLists.txt b/internal/core/thirdparty/simdjson/CMakeLists.txt index 1000c2b3cc..ddf2af9c2b 100644 --- a/internal/core/thirdparty/simdjson/CMakeLists.txt +++ b/internal/core/thirdparty/simdjson/CMakeLists.txt @@ -17,3 +17,5 @@ FetchContent_Declare( URL_HASH MD5=1b0d75ad32179c77f84f4a09d4214057 ) FetchContent_MakeAvailable(simdjson) + +set( SIMDJSON_INCLUDE_DIR ${simdjson_SOURCE_DIR}/include CACHE INTERNAL "Path to simdjson include directory" ) diff --git a/internal/core/thirdparty/tantivy/CMakeLists.txt b/internal/core/thirdparty/tantivy/CMakeLists.txt index c1435a032a..3963647eee 100644 --- a/internal/core/thirdparty/tantivy/CMakeLists.txt +++ b/internal/core/thirdparty/tantivy/CMakeLists.txt @@ -77,3 +77,5 @@ target_link_libraries(ffi_demo tantivy_binding dl ) + +set( TANTIVY_INCLUDE_DIR ${LIB_HEADER_FOLDER};${CMAKE_CURRENT_SOURCE_DIR} CACHE INTERNAL "Path to tantivy include directory" ) diff --git a/internal/core/unittest/CMakeLists.txt b/internal/core/unittest/CMakeLists.txt index 79d6516a03..aa3a64a1f7 100644 --- a/internal/core/unittest/CMakeLists.txt +++ b/internal/core/unittest/CMakeLists.txt @@ -11,6 +11,14 @@ include_directories(${CMAKE_HOME_DIRECTORY}/src) include_directories(${CMAKE_HOME_DIRECTORY}/src/thirdparty) +include_directories( + ${KNOWHERE_INCLUDE_DIR} + ${SIMDJSON_INCLUDE_DIR} + ${TANTIVY_INCLUDE_DIR} + ${CONAN_INCLUDE_DIRS} + ${MILVUS_STORAGE_INCLUDE_DIRS} + ${MILVUS_STORAGE_SRC_DIRS} +) add_definitions(-DMILVUS_TEST_SEGCORE_YAML_PATH="${CMAKE_SOURCE_DIR}/unittest/test_utils/test_segcore.yaml") @@ -128,13 +136,8 @@ if (LINUX) target_link_libraries(index_builder_test gtest - gtest_main - milvus_monitor - milvus_segcore - milvus_storage - milvus_indexbuilder - milvus_clustering - milvus_common + milvus_core + knowhere ) install(TARGETS index_builder_test DESTINATION unittest) endif() @@ -145,13 +148,8 @@ add_executable(all_tests target_link_libraries(all_tests gtest - milvus_segcore - milvus_storage - milvus_indexbuilder - milvus_clustering - pthread - milvus_common - milvus_exec + milvus_core + knowhere ) install(TARGETS all_tests DESTINATION unittest) diff --git a/internal/core/unittest/bench/CMakeLists.txt b/internal/core/unittest/bench/CMakeLists.txt index 6147e37332..68bed1644c 100644 --- a/internal/core/unittest/bench/CMakeLists.txt +++ b/internal/core/unittest/bench/CMakeLists.txt @@ -23,8 +23,8 @@ set(indexbuilder_bench_srcs add_executable(all_bench ${bench_srcs}) target_link_libraries(all_bench - milvus_segcore - milvus_log + milvus_core + knowhere pthread ) @@ -32,12 +32,9 @@ target_link_libraries(all_bench benchmark_main) add_executable(indexbuilder_bench ${indexbuilder_bench_srcs}) target_link_libraries(indexbuilder_bench - milvus_segcore - milvus_indexbuilder - milvus_index - milvus_log - pthread + milvus_core knowhere + pthread ) target_link_libraries(indexbuilder_bench benchmark_main) diff --git a/internal/core/unittest/test_azure_chunk_manager.cpp b/internal/core/unittest/test_azure_chunk_manager.cpp index ed9665e2cf..e1ac91a8ab 100644 --- a/internal/core/unittest/test_azure_chunk_manager.cpp +++ b/internal/core/unittest/test_azure_chunk_manager.cpp @@ -14,7 +14,7 @@ #include #include "common/EasyAssert.h" -#include "storage/AzureChunkManager.h" +#include "storage/azure/AzureChunkManager.h" #include "storage/Util.h" using namespace std; diff --git a/internal/indexnode/indexnode.go b/internal/indexnode/indexnode.go index f5ef580872..ed9b4ea19f 100644 --- a/internal/indexnode/indexnode.go +++ b/internal/indexnode/indexnode.go @@ -17,7 +17,7 @@ package indexnode /* -#cgo pkg-config: milvus_common milvus_indexbuilder milvus_clustering milvus_segcore +#cgo pkg-config: milvus_core #include #include diff --git a/internal/proxy/cgo_util.go b/internal/proxy/cgo_util.go index ca1336d048..1f929d29b4 100644 --- a/internal/proxy/cgo_util.go +++ b/internal/proxy/cgo_util.go @@ -17,7 +17,7 @@ package proxy /* -#cgo pkg-config: milvus_segcore +#cgo pkg-config: milvus_core #include "segcore/check_vec_index_c.h" #include */ diff --git a/internal/querynodev2/segments/cgo_util.go b/internal/querynodev2/segments/cgo_util.go index 11c9957e7d..b127a19909 100644 --- a/internal/querynodev2/segments/cgo_util.go +++ b/internal/querynodev2/segments/cgo_util.go @@ -17,7 +17,7 @@ package segments /* -#cgo pkg-config: milvus_segcore milvus_storage +#cgo pkg-config: milvus_core #include "segcore/collection_c.h" #include "common/type_c.h" diff --git a/internal/querynodev2/segments/collection.go b/internal/querynodev2/segments/collection.go index 86c85e0f80..5a5679c0c9 100644 --- a/internal/querynodev2/segments/collection.go +++ b/internal/querynodev2/segments/collection.go @@ -17,7 +17,7 @@ package segments /* -#cgo pkg-config: milvus_segcore +#cgo pkg-config: milvus_core #include "segcore/collection_c.h" #include "segcore/segment_c.h" diff --git a/internal/querynodev2/segments/index_attr_cache.go b/internal/querynodev2/segments/index_attr_cache.go index 73f1cfbe9f..d4b8fbed63 100644 --- a/internal/querynodev2/segments/index_attr_cache.go +++ b/internal/querynodev2/segments/index_attr_cache.go @@ -17,7 +17,7 @@ package segments /* -#cgo pkg-config: milvus_segcore +#cgo pkg-config: milvus_core #include "segcore/load_index_c.h" */ diff --git a/internal/querynodev2/segments/load_field_data_info.go b/internal/querynodev2/segments/load_field_data_info.go index fdca37fe86..3625c06fc2 100644 --- a/internal/querynodev2/segments/load_field_data_info.go +++ b/internal/querynodev2/segments/load_field_data_info.go @@ -17,7 +17,7 @@ package segments /* -#cgo pkg-config: milvus_segcore +#cgo pkg-config: milvus_core #include "segcore/load_field_data_c.h" */ import "C" diff --git a/internal/querynodev2/segments/load_index_info.go b/internal/querynodev2/segments/load_index_info.go index d4312b474d..36c74824b4 100644 --- a/internal/querynodev2/segments/load_index_info.go +++ b/internal/querynodev2/segments/load_index_info.go @@ -17,7 +17,7 @@ package segments /* -#cgo pkg-config: milvus_common milvus_segcore +#cgo pkg-config: milvus_core #include "segcore/load_index_c.h" #include "common/binary_set_c.h" diff --git a/internal/querynodev2/segments/manager.go b/internal/querynodev2/segments/manager.go index df90a4ac74..516986d1e5 100644 --- a/internal/querynodev2/segments/manager.go +++ b/internal/querynodev2/segments/manager.go @@ -17,7 +17,7 @@ package segments /* -#cgo pkg-config: milvus_segcore +#cgo pkg-config: milvus_core #include "segcore/collection_c.h" #include "segcore/segment_c.h" diff --git a/internal/querynodev2/segments/plan.go b/internal/querynodev2/segments/plan.go index c18a04792a..ff94ac63c9 100644 --- a/internal/querynodev2/segments/plan.go +++ b/internal/querynodev2/segments/plan.go @@ -17,7 +17,7 @@ package segments /* -#cgo pkg-config: milvus_segcore +#cgo pkg-config: milvus_core #include "segcore/collection_c.h" #include "segcore/segment_c.h" diff --git a/internal/querynodev2/segments/reduce.go b/internal/querynodev2/segments/reduce.go index 7cbaccc37b..6fcee69956 100644 --- a/internal/querynodev2/segments/reduce.go +++ b/internal/querynodev2/segments/reduce.go @@ -17,7 +17,7 @@ package segments /* -#cgo pkg-config: milvus_segcore +#cgo pkg-config: milvus_core #include "segcore/plan_c.h" #include "segcore/reduce_c.h" diff --git a/internal/querynodev2/segments/segment.go b/internal/querynodev2/segments/segment.go index 89c06225af..a1d6930427 100644 --- a/internal/querynodev2/segments/segment.go +++ b/internal/querynodev2/segments/segment.go @@ -17,7 +17,7 @@ package segments /* -#cgo pkg-config: milvus_segcore milvus_futures +#cgo pkg-config: milvus_core #include "futures/future_c.h" #include "segcore/collection_c.h" diff --git a/internal/querynodev2/segments/segment_loader.go b/internal/querynodev2/segments/segment_loader.go index 7e7e48ae9d..0a682577fb 100644 --- a/internal/querynodev2/segments/segment_loader.go +++ b/internal/querynodev2/segments/segment_loader.go @@ -17,7 +17,7 @@ package segments /* -#cgo pkg-config: milvus_segcore +#cgo pkg-config: milvus_core #include "segcore/load_index_c.h" */ diff --git a/internal/querynodev2/segments/trace.go b/internal/querynodev2/segments/trace.go index 7fb9c565bf..44ed576e2a 100644 --- a/internal/querynodev2/segments/trace.go +++ b/internal/querynodev2/segments/trace.go @@ -17,7 +17,7 @@ package segments /* -#cgo pkg-config: milvus_segcore +#cgo pkg-config: milvus_core #include "segcore/segment_c.h" */ diff --git a/internal/querynodev2/segments/utils.go b/internal/querynodev2/segments/utils.go index 8eb1eec424..14ddedd2d8 100644 --- a/internal/querynodev2/segments/utils.go +++ b/internal/querynodev2/segments/utils.go @@ -1,7 +1,7 @@ package segments /* -#cgo pkg-config: milvus_segcore milvus_common +#cgo pkg-config: milvus_core #include "segcore/collection_c.h" #include "segcore/segment_c.h" diff --git a/internal/querynodev2/server.go b/internal/querynodev2/server.go index 8994141436..62c2c5389e 100644 --- a/internal/querynodev2/server.go +++ b/internal/querynodev2/server.go @@ -17,7 +17,7 @@ package querynodev2 /* -#cgo pkg-config: milvus_segcore milvus_common +#cgo pkg-config: milvus_core #include "segcore/collection_c.h" #include "segcore/segment_c.h" diff --git a/internal/util/analyzecgowrapper/analyze.go b/internal/util/analyzecgowrapper/analyze.go index 3bfe9d3ff0..ad57fc3f71 100644 --- a/internal/util/analyzecgowrapper/analyze.go +++ b/internal/util/analyzecgowrapper/analyze.go @@ -17,7 +17,7 @@ package analyzecgowrapper /* -#cgo pkg-config: milvus_clustering +#cgo pkg-config: milvus_core #include // free #include "clustering/analyze_c.h" diff --git a/internal/util/analyzecgowrapper/helper.go b/internal/util/analyzecgowrapper/helper.go index 5b2f0b8fcc..7b8bbd564b 100644 --- a/internal/util/analyzecgowrapper/helper.go +++ b/internal/util/analyzecgowrapper/helper.go @@ -18,7 +18,7 @@ package analyzecgowrapper /* -#cgo pkg-config: milvus_common +#cgo pkg-config: milvus_core #include // free #include "common/type_c.h" diff --git a/internal/util/cgo/errors.go b/internal/util/cgo/errors.go index c0bb6e482f..11e703499c 100644 --- a/internal/util/cgo/errors.go +++ b/internal/util/cgo/errors.go @@ -1,7 +1,7 @@ package cgo /* -#cgo pkg-config: milvus_common +#cgo pkg-config: milvus_core #include "common/type_c.h" #include diff --git a/internal/util/cgo/executor.go b/internal/util/cgo/executor.go index a589513469..0c68b4d660 100644 --- a/internal/util/cgo/executor.go +++ b/internal/util/cgo/executor.go @@ -1,7 +1,7 @@ package cgo /* -#cgo pkg-config: milvus_futures +#cgo pkg-config: milvus_core #include "futures/future_c.h" */ diff --git a/internal/util/cgo/futures.go b/internal/util/cgo/futures.go index 3b6aadf454..c0a3b9885e 100644 --- a/internal/util/cgo/futures.go +++ b/internal/util/cgo/futures.go @@ -1,7 +1,7 @@ package cgo /* -#cgo pkg-config: milvus_futures +#cgo pkg-config: milvus_core #include "futures/future_c.h" #include diff --git a/internal/util/cgo/futures_test_case.go b/internal/util/cgo/futures_test_case.go index 3cc933c095..b77d0230d0 100644 --- a/internal/util/cgo/futures_test_case.go +++ b/internal/util/cgo/futures_test_case.go @@ -4,7 +4,7 @@ package cgo /* -#cgo pkg-config: milvus_futures +#cgo pkg-config: milvus_core #include "futures/future_c.h" #include diff --git a/internal/util/indexcgowrapper/build_index_info.go b/internal/util/indexcgowrapper/build_index_info.go index efef8020d0..72523a840e 100644 --- a/internal/util/indexcgowrapper/build_index_info.go +++ b/internal/util/indexcgowrapper/build_index_info.go @@ -17,7 +17,7 @@ package indexcgowrapper /* -#cgo pkg-config: milvus_indexbuilder +#cgo pkg-config: milvus_core #include // free #include "indexbuilder/index_c.h" */ diff --git a/internal/util/indexcgowrapper/helper.go b/internal/util/indexcgowrapper/helper.go index 32f79a17ba..6ebbd9009e 100644 --- a/internal/util/indexcgowrapper/helper.go +++ b/internal/util/indexcgowrapper/helper.go @@ -1,7 +1,7 @@ package indexcgowrapper /* -#cgo pkg-config: milvus_common milvus_storage +#cgo pkg-config: milvus_core #include // free #include "common/binary_set_c.h" diff --git a/internal/util/indexcgowrapper/index.go b/internal/util/indexcgowrapper/index.go index 523a2ac1ec..0e93fcf0f0 100644 --- a/internal/util/indexcgowrapper/index.go +++ b/internal/util/indexcgowrapper/index.go @@ -1,7 +1,7 @@ package indexcgowrapper /* -#cgo pkg-config: milvus_indexbuilder +#cgo pkg-config: milvus_core #include // free #include "indexbuilder/index_c.h" diff --git a/internal/util/initcore/init_core.go b/internal/util/initcore/init_core.go index 0651ef9d13..332e559bc7 100644 --- a/internal/util/initcore/init_core.go +++ b/internal/util/initcore/init_core.go @@ -17,7 +17,7 @@ package initcore /* -#cgo pkg-config: milvus_common milvus_storage milvus_segcore +#cgo pkg-config: milvus_core #include #include diff --git a/internal/util/metrics/c_registry.go b/internal/util/metrics/c_registry.go index 5505f31341..9d4963f623 100644 --- a/internal/util/metrics/c_registry.go +++ b/internal/util/metrics/c_registry.go @@ -19,7 +19,7 @@ package metrics /* -#cgo pkg-config: milvus_segcore milvus_storage milvus_common milvus_monitor +#cgo pkg-config: milvus_core #include #include "segcore/metrics_c.h" diff --git a/scripts/3rdparty_build.sh b/scripts/3rdparty_build.sh index 5a9b4ce39d..2e3e35c0e2 100644 --- a/scripts/3rdparty_build.sh +++ b/scripts/3rdparty_build.sh @@ -92,15 +92,3 @@ else bash -c "curl https://sh.rustup.rs -sSf | sh -s -- --default-toolchain=1.73 -y" || { echo 'rustup install failed'; exit 1;} source $HOME/.cargo/env fi - -echo "BUILD_OPENDAL: ${BUILD_OPENDAL}" -if [ "${BUILD_OPENDAL}" = "ON" ]; then - git clone --depth=1 --branch v0.43.0-rc.2 https://github.com/apache/opendal.git opendal - cd opendal - pushd bindings/c - cargo +1.73 build --release --verbose || { echo 'opendal_c build failed'; exit 1; } - popd - cp target/release/libopendal_c.a ${ROOT_DIR}/internal/core/output/lib/libopendal_c.a - cp bindings/c/include/opendal.h ${ROOT_DIR}/internal/core/output/include/opendal.h -fi -popd