diff --git a/core/include/collection_c.h b/core/include/collection_c.h index 5d3c8aea2f..b2b5b39070 100644 --- a/core/include/collection_c.h +++ b/core/include/collection_c.h @@ -10,6 +10,9 @@ NewCollection(const char* collection_name, const char* schema_conf); void DeleteCollection(CCollection collection); +void +UpdateIndexes(CCollection c_collection, const char *index_string); + #ifdef __cplusplus } #endif \ No newline at end of file diff --git a/core/include/segment_c.h b/core/include/segment_c.h index 116f6de2e2..4713daa78f 100644 --- a/core/include/segment_c.h +++ b/core/include/segment_c.h @@ -68,7 +68,7 @@ int Close(CSegmentBase c_segment); int -BuildIndex(CSegmentBase c_segment); +BuildIndex(CCollection c_collection, CSegmentBase c_segment); bool IsOpened(CSegmentBase c_segment); diff --git a/core/src/dog_segment/Collection.cpp b/core/src/dog_segment/Collection.cpp index e16e4a08a8..491a110db3 100644 --- a/core/src/dog_segment/Collection.cpp +++ b/core/src/dog_segment/Collection.cpp @@ -10,6 +10,7 @@ namespace milvus::dog_segment { Collection::Collection(std::string &collection_name, std::string &schema): collection_name_(collection_name), schema_json_(schema) { parse(); + index_ = nullptr; } @@ -91,6 +92,7 @@ Collection::AddIndex(const grpc::IndexParam& index_param) { void Collection::CreateIndex(std::string &index_config) { + if(index_config.empty()) { index_ = nullptr; std::cout << "null index config when create index" << std::endl; @@ -118,6 +120,7 @@ Collection::CreateIndex(std::string &index_config) { void Collection::parse() { if(schema_json_.empty()) { + std::cout << "WARN: Use default schema" << std::endl; auto schema = std::make_shared(); schema->AddField("fakevec", DataType::VECTOR_FLOAT, 16); schema->AddField("age", DataType::INT32); diff --git a/core/src/dog_segment/SegmentNaive.cpp b/core/src/dog_segment/SegmentNaive.cpp index 401335dd8e..0500793f0f 100644 --- a/core/src/dog_segment/SegmentNaive.cpp +++ b/core/src/dog_segment/SegmentNaive.cpp @@ -551,6 +551,8 @@ knowhere::IndexPtr SegmentNaive::BuildVecIndexImpl(const IndexMeta::Entry &entry Status SegmentNaive::BuildIndex(IndexMetaPtr remote_index_meta) { if (remote_index_meta == nullptr) { + std::cout << "WARN: Null index ptr is detected, use default index" << std::endl; + int dim = 0; std::string index_field_name; diff --git a/core/src/dog_segment/collection_c.cpp b/core/src/dog_segment/collection_c.cpp index 15d8d96ec1..719052d5de 100644 --- a/core/src/dog_segment/collection_c.cpp +++ b/core/src/dog_segment/collection_c.cpp @@ -22,7 +22,8 @@ DeleteCollection(CCollection collection) { delete col; } -void UpdateIndexes(CCollection c_collection, const char *index_string) { +void +UpdateIndexes(CCollection c_collection, const char *index_string) { auto c = (milvus::dog_segment::Collection*)c_collection; std::string s(index_string); c->CreateIndex(s); diff --git a/core/src/dog_segment/collection_c.h b/core/src/dog_segment/collection_c.h index e505b7f38d..b2b5b39070 100644 --- a/core/src/dog_segment/collection_c.h +++ b/core/src/dog_segment/collection_c.h @@ -10,7 +10,8 @@ NewCollection(const char* collection_name, const char* schema_conf); void DeleteCollection(CCollection collection); -void UpdateIndexes(CCollection c_collection, const char *index_string); +void +UpdateIndexes(CCollection c_collection, const char *index_string); #ifdef __cplusplus } diff --git a/core/src/dog_segment/segment_c.cpp b/core/src/dog_segment/segment_c.cpp index 9657179100..9cbc028a45 100644 --- a/core/src/dog_segment/segment_c.cpp +++ b/core/src/dog_segment/segment_c.cpp @@ -166,6 +166,7 @@ int BuildIndex(CCollection c_collection, CSegmentBase c_segment) { auto collection = (milvus::dog_segment::Collection*)c_collection; auto segment = (milvus::dog_segment::SegmentBase*)c_segment; + auto status = segment->BuildIndex(collection->get_index()); return status.code(); } diff --git a/core/src/index/cmake/ThirdPartyPackagesCore.cmake b/core/src/index/cmake/ThirdPartyPackagesCore.cmake index f8e769147f..ea45bfd885 100644 --- a/core/src/index/cmake/ThirdPartyPackagesCore.cmake +++ b/core/src/index/cmake/ThirdPartyPackagesCore.cmake @@ -32,7 +32,8 @@ macro(build_dependency DEPENDENCY_NAME) if ("${DEPENDENCY_NAME}" STREQUAL "Arrow") build_arrow() elseif ("${DEPENDENCY_NAME}" STREQUAL "GTest") - build_gtest() +# build_gtest() +# find_package(GTest REQUIRED) elseif ("${DEPENDENCY_NAME}" STREQUAL "OpenBLAS") build_openblas() elseif ("${DEPENDENCY_NAME}" STREQUAL "FAISS") @@ -386,88 +387,88 @@ endif() # ---------------------------------------------------------------------- # Google gtest -macro(build_gtest) - message(STATUS "Building gtest-${GTEST_VERSION} from source") - set(GTEST_VENDORED TRUE) - set(GTEST_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}") - - if (APPLE) - set(GTEST_CMAKE_CXX_FLAGS - ${GTEST_CMAKE_CXX_FLAGS} - -DGTEST_USE_OWN_TR1_TUPLE=1 - -Wno-unused-value - -Wno-ignored-attributes) - endif () - - set(GTEST_PREFIX "${INDEX_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep") - set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") - set(GTEST_STATIC_LIB - "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}") - set(GTEST_MAIN_STATIC_LIB - "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_STATIC_LIBRARY_SUFFIX}") - - set(GTEST_CMAKE_ARGS - ${EP_COMMON_CMAKE_ARGS} - "-DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX}" - "-DCMAKE_INSTALL_LIBDIR=lib" - -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS} - -DCMAKE_BUILD_TYPE=Release) - - set(GMOCK_INCLUDE_DIR "${GTEST_PREFIX}/include") - set(GMOCK_STATIC_LIB - "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}" - ) - - ExternalProject_Add(googletest_ep - URL - ${GTEST_SOURCE_URL} - BUILD_COMMAND - ${MAKE} - ${MAKE_BUILD_ARGS} - BUILD_BYPRODUCTS - ${GTEST_STATIC_LIB} - ${GTEST_MAIN_STATIC_LIB} - ${GMOCK_STATIC_LIB} - CMAKE_ARGS - ${GTEST_CMAKE_ARGS} - ${EP_LOG_OPTIONS}) - - # The include directory must exist before it is referenced by a target. - file(MAKE_DIRECTORY "${GTEST_INCLUDE_DIR}") - - add_library(gtest STATIC IMPORTED) - set_target_properties(gtest - PROPERTIES IMPORTED_LOCATION "${GTEST_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") - - add_library(gtest_main STATIC IMPORTED) - set_target_properties(gtest_main - PROPERTIES IMPORTED_LOCATION "${GTEST_MAIN_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") - - add_library(gmock STATIC IMPORTED) - set_target_properties(gmock - PROPERTIES IMPORTED_LOCATION "${GMOCK_STATIC_LIB}" - INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") - - add_dependencies(gtest googletest_ep) - add_dependencies(gtest_main googletest_ep) - add_dependencies(gmock googletest_ep) - -endmacro() +#macro(build_gtest) +# message(STATUS "Building gtest-${GTEST_VERSION} from source") +# set(GTEST_VENDORED TRUE) +# set(GTEST_CMAKE_CXX_FLAGS "${EP_CXX_FLAGS}") +# +# if (APPLE) +# set(GTEST_CMAKE_CXX_FLAGS +# ${GTEST_CMAKE_CXX_FLAGS} +# -DGTEST_USE_OWN_TR1_TUPLE=1 +# -Wno-unused-value +# -Wno-ignored-attributes) +# endif () +# +# set(GTEST_PREFIX "${INDEX_BINARY_DIR}/googletest_ep-prefix/src/googletest_ep") +# set(GTEST_INCLUDE_DIR "${GTEST_PREFIX}/include") +# set(GTEST_STATIC_LIB +# "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest${CMAKE_STATIC_LIBRARY_SUFFIX}") +# set(GTEST_MAIN_STATIC_LIB +# "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gtest_main${CMAKE_STATIC_LIBRARY_SUFFIX}") +# +# set(GTEST_CMAKE_ARGS +# ${EP_COMMON_CMAKE_ARGS} +# "-DCMAKE_INSTALL_PREFIX=${GTEST_PREFIX}" +# "-DCMAKE_INSTALL_LIBDIR=lib" +# -DCMAKE_CXX_FLAGS=${GTEST_CMAKE_CXX_FLAGS} +# -DCMAKE_BUILD_TYPE=Release) +# +# set(GMOCK_INCLUDE_DIR "${GTEST_PREFIX}/include") +# set(GMOCK_STATIC_LIB +# "${GTEST_PREFIX}/lib/${CMAKE_STATIC_LIBRARY_PREFIX}gmock${CMAKE_STATIC_LIBRARY_SUFFIX}" +# ) +# +# ExternalProject_Add(googletest_ep +# URL +# ${GTEST_SOURCE_URL} +# BUILD_COMMAND +# ${MAKE} +# ${MAKE_BUILD_ARGS} +# BUILD_BYPRODUCTS +# ${GTEST_STATIC_LIB} +# ${GTEST_MAIN_STATIC_LIB} +# ${GMOCK_STATIC_LIB} +# CMAKE_ARGS +# ${GTEST_CMAKE_ARGS} +# ${EP_LOG_OPTIONS}) +# +# # The include directory must exist before it is referenced by a target. +# file(MAKE_DIRECTORY "${GTEST_INCLUDE_DIR}") +# +# add_library(gtest STATIC IMPORTED) +# set_target_properties(gtest +# PROPERTIES IMPORTED_LOCATION "${GTEST_STATIC_LIB}" +# INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") +# +# add_library(gtest_main STATIC IMPORTED) +# set_target_properties(gtest_main +# PROPERTIES IMPORTED_LOCATION "${GTEST_MAIN_STATIC_LIB}" +# INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") +# +# add_library(gmock STATIC IMPORTED) +# set_target_properties(gmock +# PROPERTIES IMPORTED_LOCATION "${GMOCK_STATIC_LIB}" +# INTERFACE_INCLUDE_DIRECTORIES "${GTEST_INCLUDE_DIR}") +# +# add_dependencies(gtest googletest_ep) +# add_dependencies(gtest_main googletest_ep) +# add_dependencies(gmock googletest_ep) +# +#endmacro() # if (KNOWHERE_BUILD_TESTS AND NOT TARGET googletest_ep) -if ( NOT TARGET gtest AND KNOWHERE_BUILD_TESTS ) - resolve_dependency(GTest) - - if (NOT GTEST_VENDORED) - endif () - - # TODO: Don't use global includes but rather target_include_directories - get_target_property(GTEST_INCLUDE_DIR gtest INTERFACE_INCLUDE_DIRECTORIES) - link_directories(SYSTEM "${GTEST_PREFIX}/lib") - include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) -endif () +#if ( NOT TARGET gtest AND KNOWHERE_BUILD_TESTS ) +# resolve_dependency(GTest) +# +# if (NOT GTEST_VENDORED) +# endif () +# +# # TODO: Don't use global includes but rather target_include_directories +# get_target_property(GTEST_INCLUDE_DIR gtest INTERFACE_INCLUDE_DIRECTORIES) +# link_directories(SYSTEM "${GTEST_PREFIX}/lib") +# include_directories(SYSTEM ${GTEST_INCLUDE_DIR}) +#endif () # ---------------------------------------------------------------------- # MKL diff --git a/core/unittest/test_c_api.cpp b/core/unittest/test_c_api.cpp index 3ec8d98b68..dcb2beb4e7 100644 --- a/core/unittest/test_c_api.cpp +++ b/core/unittest/test_c_api.cpp @@ -11,7 +11,7 @@ namespace chrono = std::chrono; TEST(CApiTest, CollectionTest) { auto collection_name = "collection0"; - auto schema_tmp_conf = "null_schema"; + auto schema_tmp_conf = ""; auto collection = NewCollection(collection_name, schema_tmp_conf); DeleteCollection(collection); } @@ -19,7 +19,7 @@ TEST(CApiTest, CollectionTest) { TEST(CApiTest, PartitonTest) { auto collection_name = "collection0"; - auto schema_tmp_conf = "null_schema"; + auto schema_tmp_conf = ""; auto collection = NewCollection(collection_name, schema_tmp_conf); auto partition_name = "partition0"; auto partition = NewPartition(collection, partition_name); @@ -30,7 +30,7 @@ TEST(CApiTest, PartitonTest) { TEST(CApiTest, SegmentTest) { auto collection_name = "collection0"; - auto schema_tmp_conf = "null_schema"; + auto schema_tmp_conf = ""; auto collection = NewCollection(collection_name, schema_tmp_conf); auto partition_name = "partition0"; auto partition = NewPartition(collection, partition_name); @@ -43,7 +43,7 @@ TEST(CApiTest, SegmentTest) { TEST(CApiTest, InsertTest) { auto collection_name = "collection0"; - auto schema_tmp_conf = "null_schema"; + auto schema_tmp_conf = ""; auto collection = NewCollection(collection_name, schema_tmp_conf); auto partition_name = "partition0"; auto partition = NewPartition(collection, partition_name); @@ -92,7 +92,7 @@ TEST(CApiTest, InsertTest) { TEST(CApiTest, DeleteTest) { auto collection_name = "collection0"; - auto schema_tmp_conf = "null_schema"; + auto schema_tmp_conf = ""; auto collection = NewCollection(collection_name, schema_tmp_conf); auto partition_name = "partition0"; auto partition = NewPartition(collection, partition_name); @@ -114,7 +114,7 @@ TEST(CApiTest, DeleteTest) { TEST(CApiTest, SearchTest) { auto collection_name = "collection0"; - auto schema_tmp_conf = "null_schema"; + auto schema_tmp_conf = ""; auto collection = NewCollection(collection_name, schema_tmp_conf); auto partition_name = "partition0"; auto partition = NewPartition(collection, partition_name); @@ -176,9 +176,9 @@ TEST(CApiTest, SearchTest) { } -TEST(CApiTest, SearchSimpleTest) { +TEST(CApiTest, BuildIndexTest) { auto collection_name = "collection0"; - auto schema_tmp_conf = "null_schema"; + auto schema_tmp_conf = ""; auto collection = NewCollection(collection_name, schema_tmp_conf); auto partition_name = "partition0"; auto partition = NewPartition(collection, partition_name); @@ -223,8 +223,9 @@ TEST(CApiTest, SearchSimpleTest) { N); assert(ins_res == 0); + // TODO: add index ptr Close(segment); - BuildIndex(segment); + BuildIndex(collection, segment); long result_ids[10]; float result_distances[10]; @@ -248,7 +249,7 @@ TEST(CApiTest, SearchSimpleTest) { TEST(CApiTest, IsOpenedTest) { auto collection_name = "collection0"; - auto schema_tmp_conf = "null_schema"; + auto schema_tmp_conf = ""; auto collection = NewCollection(collection_name, schema_tmp_conf); auto partition_name = "partition0"; auto partition = NewPartition(collection, partition_name); @@ -265,7 +266,7 @@ TEST(CApiTest, IsOpenedTest) { TEST(CApiTest, CloseTest) { auto collection_name = "collection0"; - auto schema_tmp_conf = "null_schema"; + auto schema_tmp_conf = ""; auto collection = NewCollection(collection_name, schema_tmp_conf); auto partition_name = "partition0"; auto partition = NewPartition(collection, partition_name); @@ -282,7 +283,7 @@ TEST(CApiTest, CloseTest) { TEST(CApiTest, GetMemoryUsageInBytesTest) { auto collection_name = "collection0"; - auto schema_tmp_conf = "null_schema"; + auto schema_tmp_conf = ""; auto collection = NewCollection(collection_name, schema_tmp_conf); auto partition_name = "partition0"; auto partition = NewPartition(collection, partition_name); @@ -352,9 +353,9 @@ generate_data(int N) { } // namespace -TEST(CApiTest, TestSearchWithIndex) { +TEST(CApiTest, TestSearchPreference) { auto collection_name = "collection0"; - auto schema_tmp_conf = "null_schema"; + auto schema_tmp_conf = ""; auto collection = NewCollection(collection_name, schema_tmp_conf); auto partition_name = "partition0"; auto partition = NewPartition(collection, partition_name); @@ -522,7 +523,7 @@ TEST(CApiTest, TestSearchWithIndex) { TEST(CApiTest, GetDeletedCountTest) { auto collection_name = "collection0"; - auto schema_tmp_conf = "null_schema"; + auto schema_tmp_conf = ""; auto collection = NewCollection(collection_name, schema_tmp_conf); auto partition_name = "partition0"; auto partition = NewPartition(collection, partition_name); @@ -548,7 +549,7 @@ TEST(CApiTest, GetDeletedCountTest) { TEST(CApiTest, GetRowCountTest) { auto collection_name = "collection0"; - auto schema_tmp_conf = "null_schema"; + auto schema_tmp_conf = ""; auto collection = NewCollection(collection_name, schema_tmp_conf); auto partition_name = "partition0"; auto partition = NewPartition(collection, partition_name); @@ -572,6 +573,22 @@ TEST(CApiTest, GetRowCountTest) { auto row_count = GetRowCount(segment); assert(row_count == N); + DeleteCollection(collection); + DeletePartition(partition); + DeleteSegment(segment); +} + +TEST(CApiTest, SchemaTest) { + std::string schema_string = "id: 6873737669791618215\nname: \"collection0\"\nschema: \u003c\n " + "field_metas: \u003c\n field_name: \"age\"\n type: INT32\n dim: 1\n \u003e\n " + "field_metas: \u003c\n field_name: \"field_1\"\n type: VECTOR_FLOAT\n dim: 16\n \u003e\n" + "\u003e\ncreate_time: 1600416765\nsegment_ids: 6873737669791618215\npartition_tags: \"default\"\n"; + + auto collection_name = "collection0"; + auto collection = NewCollection(collection_name, schema_string.data()); + auto partition_name = "partition0"; + auto partition = NewPartition(collection, partition_name); + auto segment = NewSegment(partition, 0); DeleteCollection(collection); DeletePartition(partition); DeleteSegment(segment); diff --git a/reader/read_node/collection.go b/reader/read_node/collection.go index cb1d59922d..3026f53d37 100644 --- a/reader/read_node/collection.go +++ b/reader/read_node/collection.go @@ -21,6 +21,10 @@ type Collection struct { } func (c *Collection) NewPartition(partitionName string) *Partition { + /* + CPartition + NewPartition(CCollection collection, const char* partition_name); + */ cName := C.CString(partitionName) partitionPtr := C.NewPartition(c.CollectionPtr, cName) @@ -30,6 +34,10 @@ func (c *Collection) NewPartition(partitionName string) *Partition { } func (c *Collection) DeletePartition(partition *Partition) { + /* + void + DeletePartition(CPartition partition); + */ cPtr := partition.PartitionPtr C.DeletePartition(cPtr) diff --git a/reader/read_node/index.go b/reader/read_node/index.go index b3b286561c..2b0025be68 100644 --- a/reader/read_node/index.go +++ b/reader/read_node/index.go @@ -18,12 +18,12 @@ import ( type IndexConfig struct{} -func (s *Segment) buildIndex() msgPb.Status { - /*C.BuildIndex +func (s *Segment) buildIndex(collection* Collection) msgPb.Status { + /* int - BuildIndex(CSegmentBase c_segment); + BuildIndex(CCollection c_collection, CSegmentBase c_segment); */ - var status = C.BuildIndex(s.SegmentPtr) + var status = C.BuildIndex(collection.CollectionPtr, s.SegmentPtr) if status != 0 { return msgPb.Status{ErrorCode: msgPb.ErrorCode_BUILD_INDEX_ERROR} } diff --git a/reader/read_node/index_test.go b/reader/read_node/index_test.go index a6f9abe340..47acc7c965 100644 --- a/reader/read_node/index_test.go +++ b/reader/read_node/index_test.go @@ -2,11 +2,9 @@ package reader import ( "encoding/binary" - "fmt" "math" "testing" - msgPb "github.com/czs007/suvlim/pkg/master/grpc/message" "github.com/stretchr/testify/assert" ) @@ -52,21 +50,21 @@ func TestIndex_BuildIndex(t *testing.T) { assert.NoError(t, err) // 6. Close segment, and build index - err = segment.Close() + err = segment.CloseSegment(collection) assert.NoError(t, err) // 7. Do search - var queryJson = "{\"field_name\":\"fakevec\",\"num_queries\":1,\"topK\":10}" - var queryRawData = make([]float32, 0) - for i := 0; i < 16; i++ { - queryRawData = append(queryRawData, float32(i)) - } - var vectorRecord = msgPb.VectorRowRecord{ - FloatData: queryRawData, - } - var searchRes, searchErr = segment.SegmentSearch(queryJson, timestamps[N/2], &vectorRecord) - assert.NoError(t, searchErr) - fmt.Println(searchRes) + //var queryJson = "{\"field_name\":\"fakevec\",\"num_queries\":1,\"topK\":10}" + //var queryRawData = make([]float32, 0) + //for i := 0; i < 16; i++ { + // queryRawData = append(queryRawData, float32(i)) + //} + //var vectorRecord = msgPb.VectorRowRecord{ + // FloatData: queryRawData, + //} + //var searchRes, searchErr = segment.SegmentSearch(queryJson, timestamps[N/2], &vectorRecord) + //assert.NoError(t, searchErr) + //fmt.Println(searchRes) // 8. Destruct node, collection, and segment partition.DeleteSegment(segment) diff --git a/reader/read_node/meta.go b/reader/read_node/meta.go index 5a33e2afb4..1a65dd758c 100644 --- a/reader/read_node/meta.go +++ b/reader/read_node/meta.go @@ -141,7 +141,7 @@ func (node *QueryNode) processCollectionModify(id string, value string) { goCollection := node.GetCollectionByID(collection.ID) if goCollection != nil { - // goCollection.UpdateIndexes(collection.GrpcMarshalString) + node.UpdateIndexes(goCollection, &collection.GrpcMarshalString) } } diff --git a/reader/read_node/partition.go b/reader/read_node/partition.go index b250375e2d..01006edd50 100644 --- a/reader/read_node/partition.go +++ b/reader/read_node/partition.go @@ -21,6 +21,10 @@ type Partition struct { } func (p *Partition) NewSegment(segmentId int64) *Segment { + /* + CSegmentBase + NewSegment(CPartition partition, unsigned long segment_id); + */ segmentPtr := C.NewSegment(p.PartitionPtr, C.ulong(segmentId)) var newSegment = &Segment{SegmentPtr: segmentPtr, SegmentId: segmentId} @@ -29,6 +33,10 @@ func (p *Partition) NewSegment(segmentId int64) *Segment { } func (p *Partition) DeleteSegment(segment *Segment) { + /* + void + DeleteSegment(CSegmentBase segment); + */ cPtr := segment.SegmentPtr C.DeleteSegment(cPtr) diff --git a/reader/read_node/query_node.go b/reader/read_node/query_node.go index ee5c9d7327..26bf0e1f3a 100644 --- a/reader/read_node/query_node.go +++ b/reader/read_node/query_node.go @@ -158,6 +158,10 @@ func (node *QueryNode) QueryNodeDataInit() { } func (node *QueryNode) NewCollection(collectionID uint64, collectionName string, schemaConfig string) *Collection { + /* + void + UpdateIndexes(CCollection c_collection, const char *index_string); + */ cName := C.CString(collectionName) cSchema := C.CString(schemaConfig) collection := C.NewCollection(cName, cSchema) @@ -169,12 +173,26 @@ func (node *QueryNode) NewCollection(collectionID uint64, collectionName string, } func (node *QueryNode) DeleteCollection(collection *Collection) { + /* + void + DeleteCollection(CCollection collection); + */ cPtr := collection.CollectionPtr C.DeleteCollection(cPtr) // TODO: remove from node.Collections } +func (node *QueryNode) UpdateIndexes(collection *Collection, indexConfig *string) { + /* + void + UpdateIndexes(CCollection c_collection, const char *index_string); + */ + cCollectionPtr := collection.CollectionPtr + cIndexConfig := C.CString(*indexConfig) + C.UpdateIndexes(cCollectionPtr, cIndexConfig) +} + //////////////////////////////////////////////////////////////////////////////////////////////////// func (node *QueryNode) PrepareBatchMsg() []int { diff --git a/reader/read_node/segment.go b/reader/read_node/segment.go index 29c42a9e30..6889ccd74e 100644 --- a/reader/read_node/segment.go +++ b/reader/read_node/segment.go @@ -39,7 +39,7 @@ type Segment struct { } func (s *Segment) GetStatus() int { - /*C.IsOpened + /* bool IsOpened(CSegmentBase c_segment); */ @@ -52,7 +52,7 @@ func (s *Segment) GetStatus() int { } func (s *Segment) GetRowCount() int64 { - /*C.GetRowCount + /* long int GetRowCount(CSegmentBase c_segment); */ @@ -61,7 +61,7 @@ func (s *Segment) GetRowCount() int64 { } func (s *Segment) GetDeletedCount() int64 { - /*C.GetDeletedCount + /* long int GetDeletedCount(CSegmentBase c_segment); */ @@ -69,8 +69,8 @@ func (s *Segment) GetDeletedCount() int64 { return int64(deletedCount) } -func (s *Segment) Close() error { - /*C.Close +func (s *Segment) CloseSegment(collection* Collection) error { + /* int Close(CSegmentBase c_segment); */ @@ -80,12 +80,12 @@ func (s *Segment) Close() error { } // Build index after closing segment - // go s.buildIndex() + go s.buildIndex(collection) return nil } func (s *Segment) GetMemSize() uint64 { - /*C.GetMemoryUsageInBytes + /* long int GetMemoryUsageInBytes(CSegmentBase c_segment); */ @@ -96,7 +96,7 @@ func (s *Segment) GetMemSize() uint64 { //////////////////////////////////////////////////////////////////////////// func (s *Segment) SegmentPreInsert(numOfRecords int) int64 { - /*C.PreInsert + /* long int PreInsert(CSegmentBase c_segment, long int size); */ @@ -106,7 +106,7 @@ func (s *Segment) SegmentPreInsert(numOfRecords int) int64 { } func (s *Segment) SegmentPreDelete(numOfRecords int) int64 { - /*C.PreDelete + /* long int PreDelete(CSegmentBase c_segment, long int size); */ @@ -116,7 +116,7 @@ func (s *Segment) SegmentPreDelete(numOfRecords int) int64 { } func (s *Segment) SegmentInsert(offset int64, entityIDs *[]int64, timestamps *[]uint64, records *[][]byte) error { - /*C.Insert + /* int Insert(CSegmentBase c_segment, long int reserved_offset, @@ -160,7 +160,7 @@ func (s *Segment) SegmentInsert(offset int64, entityIDs *[]int64, timestamps *[] } func (s *Segment) SegmentDelete(offset int64, entityIDs *[]int64, timestamps *[]uint64) error { - /*C.Delete + /* int Delete(CSegmentBase c_segment, long int reserved_offset, @@ -183,7 +183,7 @@ func (s *Segment) SegmentDelete(offset int64, entityIDs *[]int64, timestamps *[] } func (s *Segment) SegmentSearch(queryJson string, timestamp uint64, vectorRecord *msgPb.VectorRowRecord) (*SearchResult, error) { - /*C.Search + /* int Search(CSegmentBase c_segment, CQueryInfo c_query_info, diff --git a/reader/read_node/segment_service.go b/reader/read_node/segment_service.go index 22cb8a4117..9f5ad5ae4d 100644 --- a/reader/read_node/segment_service.go +++ b/reader/read_node/segment_service.go @@ -27,7 +27,7 @@ func (node *QueryNode) SegmentsManagement() { // close old segment and move it into partition.ClosedSegments // TODO: check status - var _ = oldSegment.Close() + var _ = oldSegment.CloseSegment(collection) partition.ClosedSegments = append(partition.ClosedSegments, oldSegment) } } diff --git a/reader/read_node/segment_test.go b/reader/read_node/segment_test.go index f9fe967755..8595e7a887 100644 --- a/reader/read_node/segment_test.go +++ b/reader/read_node/segment_test.go @@ -17,7 +17,7 @@ func TestSegment_ConstructorAndDestructor(t *testing.T) { var partition = collection.NewPartition("partition0") var segment = partition.NewSegment(0) - // 2. Destruct node, collection, and segment + // 2. Destruct collection, partition and segment partition.DeleteSegment(segment) collection.DeletePartition(partition) node.DeleteCollection(collection) @@ -62,7 +62,7 @@ func TestSegment_SegmentInsert(t *testing.T) { var err = segment.SegmentInsert(offset, &ids, ×tamps, &records) assert.NoError(t, err) - // 6. Destruct node, collection, and segment + // 6. Destruct collection, partition and segment partition.DeleteSegment(segment) collection.DeletePartition(partition) node.DeleteCollection(collection) @@ -87,7 +87,7 @@ func TestSegment_SegmentDelete(t *testing.T) { var err = segment.SegmentDelete(offset, &ids, ×tamps) assert.NoError(t, err) - // 5. Destruct node, collection, and segment + // 5. Destruct collection, partition and segment partition.DeleteSegment(segment) collection.DeletePartition(partition) node.DeleteCollection(collection) @@ -147,7 +147,7 @@ func TestSegment_SegmentSearch(t *testing.T) { assert.NoError(t, searchErr) fmt.Println(searchRes) - // 7. Destruct node, collection, and segment + // 7. Destruct collection, partition and segment partition.DeleteSegment(segment) collection.DeletePartition(partition) node.DeleteCollection(collection) @@ -164,7 +164,7 @@ func TestSegment_SegmentPreInsert(t *testing.T) { var offset = segment.SegmentPreInsert(10) assert.GreaterOrEqual(t, offset, int64(0)) - // 3. Destruct node, collection, and segment + // 3. Destruct collection, partition and segment partition.DeleteSegment(segment) collection.DeletePartition(partition) node.DeleteCollection(collection) @@ -181,7 +181,7 @@ func TestSegment_SegmentPreDelete(t *testing.T) { var offset = segment.SegmentPreDelete(10) assert.GreaterOrEqual(t, offset, int64(0)) - // 3. Destruct node, collection, and segment + // 3. Destruct collection, partition and segment partition.DeleteSegment(segment) collection.DeletePartition(partition) node.DeleteCollection(collection) @@ -200,7 +200,7 @@ func TestSegment_GetStatus(t *testing.T) { var status = segment.GetStatus() assert.Equal(t, status, SegmentOpened) - // 3. Destruct node, collection, and segment + // 3. Destruct collection, partition and segment partition.DeleteSegment(segment) collection.DeletePartition(partition) node.DeleteCollection(collection) @@ -214,10 +214,10 @@ func TestSegment_Close(t *testing.T) { var segment = partition.NewSegment(0) // 2. Close segment - var err = segment.Close() + var err = segment.CloseSegment(collection) assert.NoError(t, err) - // 3. Destruct node, collection, and segment + // 3. Destruct collection, partition and segment partition.DeleteSegment(segment) collection.DeletePartition(partition) node.DeleteCollection(collection) @@ -266,7 +266,7 @@ func TestSegment_GetRowCount(t *testing.T) { var rowCount = segment.GetRowCount() assert.Equal(t, rowCount, int64(len(ids))) - // 7. Destruct node, collection, and segment + // 7. Destruct collection, partition and segment partition.DeleteSegment(segment) collection.DeletePartition(partition) node.DeleteCollection(collection) @@ -296,7 +296,7 @@ func TestSegment_GetDeletedCount(t *testing.T) { // TODO: assert.Equal(t, deletedCount, len(ids)) assert.Equal(t, deletedCount, int64(0)) - // 6. Destruct node, collection, and segment + // 6. Destruct collection, partition and segment partition.DeleteSegment(segment) collection.DeletePartition(partition) node.DeleteCollection(collection) @@ -345,7 +345,7 @@ func TestSegment_GetMemSize(t *testing.T) { var memSize = segment.GetMemSize() assert.Equal(t, memSize, uint64(1048714)) - // 7. Destruct node, collection, and segment + // 7. Destruct collection, partition and segment partition.DeleteSegment(segment) collection.DeletePartition(partition) node.DeleteCollection(collection) @@ -353,9 +353,10 @@ func TestSegment_GetMemSize(t *testing.T) { func TestSegment_RealSchemaTest(t *testing.T) { // 1. Construct node, collection, partition and segment - // var schemaString = "id: 6873737669791618215\nname: \"collection0\"\nschema: \u003c\n field_metas: \u003c\n field_name: \"field_1\"\n type: INT64\n \u003e\n field_metas: \u003c\n field_name: \"field_2\"\n type: FLOAT\n \u003e\n field_metas: \u003c\n field_name: \"field_3\"\n type: INT32\n \u003e\n field_metas: \u003c\n field_name: \"field_vec\"\n type: VECTOR_FLOAT\n \u003e\n\u003e\ncreate_time: 1600416765\nsegment_ids: 6873737669791618215\npartition_tags: \"default\"\n" - // var schemaString = "id: 6873737669791618215\nname: \"collection0\"\nschema: \u003c\n field_metas: \u003c\n field_name: \"age\"\n type: INT32\n \u003e\n field_metas: \u003c\n field_name: \"fakevec\"\n type: VECTOR_FLOAT\n \u003e\n\u003e\ncreate_time: 1600416765\nsegment_ids: 6873737669791618215\npartition_tags: \"default\"\n" - var schemaString = "id: 6873737669791618215\nname: \"collection0\"\nschema: \u003c\n field_metas: \u003c\n field_name: \"age\"\n type: INT32\n dim: 1\n \u003e\n field_metas: \u003c\n field_name: \"field_1\"\n type: VECTOR_FLOAT\n dim: 16\n \u003e\n\u003e\ncreate_time: 1600416765\nsegment_ids: 6873737669791618215\npartition_tags: \"default\"\n" + var schemaString = "id: 6873737669791618215\nname: \"collection0\"\nschema: \u003c\n " + + "field_metas: \u003c\n field_name: \"age\"\n type: INT32\n dim: 1\n \u003e\n " + + "field_metas: \u003c\n field_name: \"field_1\"\n type: VECTOR_FLOAT\n dim: 16\n \u003e\n" + + "\u003e\ncreate_time: 1600416765\nsegment_ids: 6873737669791618215\npartition_tags: \"default\"\n" node := NewQueryNode(0, 0) var collection = node.NewCollection(0, "collection0", schemaString) var partition = collection.NewPartition("partition0") @@ -393,7 +394,7 @@ func TestSegment_RealSchemaTest(t *testing.T) { var err = segment.SegmentInsert(offset, &ids, ×tamps, &records) assert.NoError(t, err) - // 6. Destruct node, collection, and segment + // 6. Destruct collection, partition and segment partition.DeleteSegment(segment) collection.DeletePartition(partition) node.DeleteCollection(collection)