diff --git a/internal/core/src/segcore/CMakeLists.txt b/internal/core/src/segcore/CMakeLists.txt index 8298c8a87d..74c916efbb 100644 --- a/internal/core/src/segcore/CMakeLists.txt +++ b/internal/core/src/segcore/CMakeLists.txt @@ -5,9 +5,7 @@ set(SEGCORE_FILES IndexMeta.cpp ConcurrentVector.cpp Collection.cpp - Partition.cpp collection_c.cpp - partition_c.cpp segment_c.cpp SegmentBase.cpp IndexingEntry.cpp diff --git a/internal/core/src/segcore/Collection.cpp b/internal/core/src/segcore/Collection.cpp index 7625791d68..7c8c623f5f 100644 --- a/internal/core/src/segcore/Collection.cpp +++ b/internal/core/src/segcore/Collection.cpp @@ -8,8 +8,8 @@ namespace milvus::segcore { -Collection::Collection(std::string& collection_name, std::string& schema) - : collection_name_(collection_name), schema_json_(schema) { +Collection::Collection(const std::string& collection_proto) + : collection_proto_(collection_proto) { parse(); index_ = nullptr; } @@ -112,7 +112,7 @@ Collection::CreateIndex(std::string& index_config) { void Collection::parse() { - if (schema_json_.empty()) { + if (collection_proto_.empty()) { std::cout << "WARN: Use default schema" << std::endl; auto schema = std::make_shared(); schema->AddField("fakevec", DataType::VECTOR_FLOAT, 16); @@ -122,11 +122,16 @@ Collection::parse() { } milvus::proto::etcd::CollectionMeta collection_meta; - auto suc = google::protobuf::TextFormat::ParseFromString(schema_json_, &collection_meta); + auto suc = google::protobuf::TextFormat::ParseFromString(collection_proto_, &collection_meta); if (!suc) { std::cerr << "unmarshal schema string failed" << std::endl; } + + collection_name_ = collection_meta.schema().name(); + // TODO: delete print + std::cout << "create collection " << collection_meta.schema().name() << std::endl; + auto schema = std::make_shared(); for (const milvus::proto::schema::FieldSchema& child : collection_meta.schema().fields()) { const auto& type_params = child.type_params(); diff --git a/internal/core/src/segcore/Collection.h b/internal/core/src/segcore/Collection.h index b50564f557..3518d67b26 100644 --- a/internal/core/src/segcore/Collection.h +++ b/internal/core/src/segcore/Collection.h @@ -1,13 +1,13 @@ #pragma once -#include "segcore/Partition.h" #include "SegmentDefs.h" +#include "IndexMeta.h" namespace milvus::segcore { class Collection { public: - explicit Collection(std::string& collection_name, std::string& schema); + explicit Collection(const std::string& collection_proto); void parse(); @@ -23,7 +23,7 @@ class Collection { return index_; } - std::string& + const std::string& get_collection_name() { return collection_name_; } @@ -31,7 +31,7 @@ class Collection { private: IndexMetaPtr index_; std::string collection_name_; - std::string schema_json_; + std::string collection_proto_; SchemaPtr schema_; }; diff --git a/internal/core/src/segcore/Partition.cpp b/internal/core/src/segcore/Partition.cpp deleted file mode 100644 index 015a754b5e..0000000000 --- a/internal/core/src/segcore/Partition.cpp +++ /dev/null @@ -1,9 +0,0 @@ -#include "Partition.h" - -namespace milvus::segcore { - -Partition::Partition(std::string& partition_name, SchemaPtr& schema, IndexMetaPtr& index) - : partition_name_(partition_name), schema_(schema), index_(index) { -} - -} // namespace milvus::segcore diff --git a/internal/core/src/segcore/Partition.h b/internal/core/src/segcore/Partition.h deleted file mode 100644 index f7235664ee..0000000000 --- a/internal/core/src/segcore/Partition.h +++ /dev/null @@ -1,35 +0,0 @@ -#pragma once - -#include "SegmentBase.h" - -namespace milvus::segcore { - -class Partition { - public: - explicit Partition(std::string& partition_name, SchemaPtr& schema, IndexMetaPtr& index); - - public: - SchemaPtr& - get_schema() { - return schema_; - } - - IndexMetaPtr& - get_index() { - return index_; - } - - std::string& - get_partition_name() { - return partition_name_; - } - - private: - std::string partition_name_; - SchemaPtr schema_; - IndexMetaPtr index_; -}; - -using PartitionPtr = std::unique_ptr; - -} // namespace milvus::segcore \ No newline at end of file diff --git a/internal/core/src/segcore/collection_c.cpp b/internal/core/src/segcore/collection_c.cpp index 120f11f59c..95db2d70af 100644 --- a/internal/core/src/segcore/collection_c.cpp +++ b/internal/core/src/segcore/collection_c.cpp @@ -1,15 +1,13 @@ +#include #include "collection_c.h" #include "Collection.h" CCollection -NewCollection(const char* collection_name, const char* schema_conf) { - auto name = std::string(collection_name); - auto conf = std::string(schema_conf); +NewCollection(const char* collection_proto) { + auto proto = std::string(collection_proto); - auto collection = std::make_unique(name, conf); + auto collection = std::make_unique(proto); - // TODO: delete print - std::cout << "create collection " << collection_name << std::endl; return (void*)collection.release(); } diff --git a/internal/core/src/segcore/collection_c.h b/internal/core/src/segcore/collection_c.h index 5d3c8aea2f..3824e9d24a 100644 --- a/internal/core/src/segcore/collection_c.h +++ b/internal/core/src/segcore/collection_c.h @@ -5,7 +5,7 @@ extern "C" { typedef void* CCollection; CCollection -NewCollection(const char* collection_name, const char* schema_conf); +NewCollection(const char* collection_proto); void DeleteCollection(CCollection collection); diff --git a/internal/core/src/segcore/partition_c.cpp b/internal/core/src/segcore/partition_c.cpp deleted file mode 100644 index 36c3473543..0000000000 --- a/internal/core/src/segcore/partition_c.cpp +++ /dev/null @@ -1,29 +0,0 @@ -#include "partition_c.h" -#include "Partition.h" -#include "Collection.h" - -CPartition -NewPartition(CCollection collection, const char* partition_name) { - auto c = (milvus::segcore::Collection*)collection; - - auto name = std::string(partition_name); - - auto schema = c->get_schema(); - - auto index = c->get_index(); - - auto partition = std::make_unique(name, schema, index); - - // TODO: delete print - std::cout << "create partition " << name << std::endl; - return (void*)partition.release(); -} - -void -DeletePartition(CPartition partition) { - auto p = (milvus::segcore::Partition*)partition; - - // TODO: delete print - std::cout << "delete partition " << p->get_partition_name() << std::endl; - delete p; -} diff --git a/internal/core/src/segcore/partition_c.h b/internal/core/src/segcore/partition_c.h deleted file mode 100644 index d1bfeead05..0000000000 --- a/internal/core/src/segcore/partition_c.h +++ /dev/null @@ -1,17 +0,0 @@ -#ifdef __cplusplus -extern "C" { -#endif - -#include "collection_c.h" - -typedef void* CPartition; - -CPartition -NewPartition(CCollection collection, const char* partition_name); - -void -DeletePartition(CPartition partition); - -#ifdef __cplusplus -} -#endif \ No newline at end of file diff --git a/internal/core/src/segcore/segment_c.cpp b/internal/core/src/segcore/segment_c.cpp index 359ad6a74c..26d5c7cb0e 100644 --- a/internal/core/src/segcore/segment_c.cpp +++ b/internal/core/src/segcore/segment_c.cpp @@ -3,16 +3,15 @@ #include "SegmentBase.h" #include "Collection.h" #include "segment_c.h" -#include "Partition.h" #include #include #include CSegmentBase -NewSegment(CPartition partition, unsigned long segment_id) { - auto p = (milvus::segcore::Partition*)partition; +NewSegment(CCollection collection, unsigned long segment_id) { + auto col = (milvus::segcore::Collection*)collection; - auto segment = milvus::segcore::CreateSegment(p->get_schema()); + auto segment = milvus::segcore::CreateSegment(col->get_schema()); // TODO: delete print std::cout << "create segment " << segment_id << std::endl; diff --git a/internal/core/src/segcore/segment_c.h b/internal/core/src/segcore/segment_c.h index 6279e486f7..7f2c505f9e 100644 --- a/internal/core/src/segcore/segment_c.h +++ b/internal/core/src/segcore/segment_c.h @@ -3,7 +3,7 @@ extern "C" { #endif #include -#include "partition_c.h" +#include "collection_c.h" typedef void* CSegmentBase; @@ -14,7 +14,7 @@ typedef struct CQueryInfo { } CQueryInfo; CSegmentBase -NewSegment(CPartition partition, unsigned long segment_id); +NewSegment(CCollection collection, unsigned long segment_id); void DeleteSegment(CSegmentBase segment); diff --git a/internal/core/unittest/test_c_api.cpp b/internal/core/unittest/test_c_api.cpp index 5d4d47ba1f..07293b28b0 100644 --- a/internal/core/unittest/test_c_api.cpp +++ b/internal/core/unittest/test_c_api.cpp @@ -10,41 +10,23 @@ namespace chrono = std::chrono; TEST(CApiTest, CollectionTest) { - auto collection_name = "collection0"; auto schema_tmp_conf = ""; - auto collection = NewCollection(collection_name, schema_tmp_conf); + auto collection = NewCollection(schema_tmp_conf); DeleteCollection(collection); } -TEST(CApiTest, PartitonTest) { - auto collection_name = "collection0"; - auto schema_tmp_conf = ""; - auto collection = NewCollection(collection_name, schema_tmp_conf); - auto partition_name = "partition0"; - auto partition = NewPartition(collection, partition_name); - DeleteCollection(collection); - DeletePartition(partition); -} - TEST(CApiTest, SegmentTest) { - auto collection_name = "collection0"; auto schema_tmp_conf = ""; - auto collection = NewCollection(collection_name, schema_tmp_conf); - auto partition_name = "partition0"; - auto partition = NewPartition(collection, partition_name); - auto segment = NewSegment(partition, 0); + auto collection = NewCollection(schema_tmp_conf); + auto segment = NewSegment(collection, 0); DeleteCollection(collection); - DeletePartition(partition); DeleteSegment(segment); } TEST(CApiTest, InsertTest) { - auto collection_name = "collection0"; auto schema_tmp_conf = ""; - auto collection = NewCollection(collection_name, schema_tmp_conf); - auto partition_name = "partition0"; - auto partition = NewPartition(collection, partition_name); - auto segment = NewSegment(partition, 0); + auto collection = NewCollection(schema_tmp_conf); + auto segment = NewSegment(collection, 0); std::vector raw_data; std::vector timestamps; @@ -73,17 +55,13 @@ TEST(CApiTest, InsertTest) { assert(res == 0); DeleteCollection(collection); - DeletePartition(partition); DeleteSegment(segment); } TEST(CApiTest, DeleteTest) { - auto collection_name = "collection0"; auto schema_tmp_conf = ""; - auto collection = NewCollection(collection_name, schema_tmp_conf); - auto partition_name = "partition0"; - auto partition = NewPartition(collection, partition_name); - auto segment = NewSegment(partition, 0); + auto collection = NewCollection(schema_tmp_conf); + auto segment = NewSegment(collection, 0); long delete_row_ids[] = {100000, 100001, 100002}; unsigned long delete_timestamps[] = {0, 0, 0}; @@ -94,17 +72,13 @@ TEST(CApiTest, DeleteTest) { assert(del_res == 0); DeleteCollection(collection); - DeletePartition(partition); DeleteSegment(segment); } TEST(CApiTest, SearchTest) { - auto collection_name = "collection0"; auto schema_tmp_conf = ""; - auto collection = NewCollection(collection_name, schema_tmp_conf); - auto partition_name = "partition0"; - auto partition = NewPartition(collection, partition_name); - auto segment = NewSegment(partition, 0); + auto collection = NewCollection(schema_tmp_conf); + auto segment = NewSegment(collection, 0); std::vector raw_data; std::vector timestamps; @@ -146,17 +120,13 @@ TEST(CApiTest, SearchTest) { assert(sea_res == 0); DeleteCollection(collection); - DeletePartition(partition); DeleteSegment(segment); } TEST(CApiTest, BuildIndexTest) { - auto collection_name = "collection0"; auto schema_tmp_conf = ""; - auto collection = NewCollection(collection_name, schema_tmp_conf); - auto partition_name = "partition0"; - auto partition = NewPartition(collection, partition_name); - auto segment = NewSegment(partition, 0); + auto collection = NewCollection(schema_tmp_conf); + auto segment = NewSegment(collection, 0); std::vector raw_data; std::vector timestamps; @@ -205,49 +175,37 @@ TEST(CApiTest, BuildIndexTest) { assert(sea_res == 0); DeleteCollection(collection); - DeletePartition(partition); DeleteSegment(segment); } TEST(CApiTest, IsOpenedTest) { - auto collection_name = "collection0"; auto schema_tmp_conf = ""; - auto collection = NewCollection(collection_name, schema_tmp_conf); - auto partition_name = "partition0"; - auto partition = NewPartition(collection, partition_name); - auto segment = NewSegment(partition, 0); + auto collection = NewCollection(schema_tmp_conf); + auto segment = NewSegment(collection, 0); auto is_opened = IsOpened(segment); assert(is_opened); DeleteCollection(collection); - DeletePartition(partition); DeleteSegment(segment); } TEST(CApiTest, CloseTest) { - auto collection_name = "collection0"; auto schema_tmp_conf = ""; - auto collection = NewCollection(collection_name, schema_tmp_conf); - auto partition_name = "partition0"; - auto partition = NewPartition(collection, partition_name); - auto segment = NewSegment(partition, 0); + auto collection = NewCollection(schema_tmp_conf); + auto segment = NewSegment(collection, 0); auto status = Close(segment); assert(status == 0); DeleteCollection(collection); - DeletePartition(partition); DeleteSegment(segment); } TEST(CApiTest, GetMemoryUsageInBytesTest) { - auto collection_name = "collection0"; auto schema_tmp_conf = ""; - auto collection = NewCollection(collection_name, schema_tmp_conf); - auto partition_name = "partition0"; - auto partition = NewPartition(collection, partition_name); - auto segment = NewSegment(partition, 0); + auto collection = NewCollection(schema_tmp_conf); + auto segment = NewSegment(collection, 0); auto old_memory_usage_size = GetMemoryUsageInBytes(segment); std::cout << "old_memory_usage_size = " << old_memory_usage_size << std::endl; @@ -285,7 +243,6 @@ TEST(CApiTest, GetMemoryUsageInBytesTest) { assert(memory_usage_size == 2785280); DeleteCollection(collection); - DeletePartition(partition); DeleteSegment(segment); } @@ -315,12 +272,9 @@ generate_data(int N) { } // namespace TEST(CApiTest, TestSearchPreference) { - auto collection_name = "collection0"; auto schema_tmp_conf = ""; - auto collection = NewCollection(collection_name, schema_tmp_conf); - auto partition_name = "partition0"; - auto partition = NewPartition(collection, partition_name); - auto segment = NewSegment(partition, 0); + auto collection = NewCollection(schema_tmp_conf); + auto segment = NewSegment(collection, 0); auto beg = chrono::high_resolution_clock::now(); auto next = beg; @@ -432,17 +386,13 @@ TEST(CApiTest, TestSearchPreference) { // } DeleteCollection(collection); - DeletePartition(partition); DeleteSegment(segment); } TEST(CApiTest, GetDeletedCountTest) { - auto collection_name = "collection0"; auto schema_tmp_conf = ""; - auto collection = NewCollection(collection_name, schema_tmp_conf); - auto partition_name = "partition0"; - auto partition = NewPartition(collection, partition_name); - auto segment = NewSegment(partition, 0); + auto collection = NewCollection(schema_tmp_conf); + auto segment = NewSegment(collection, 0); long delete_row_ids[] = {100000, 100001, 100002}; unsigned long delete_timestamps[] = {0, 0, 0}; @@ -457,17 +407,13 @@ TEST(CApiTest, GetDeletedCountTest) { assert(deleted_count == 0); DeleteCollection(collection); - DeletePartition(partition); DeleteSegment(segment); } TEST(CApiTest, GetRowCountTest) { - auto collection_name = "collection0"; auto schema_tmp_conf = ""; - auto collection = NewCollection(collection_name, schema_tmp_conf); - auto partition_name = "partition0"; - auto partition = NewPartition(collection, partition_name); - auto segment = NewSegment(partition, 0); + auto collection = NewCollection(schema_tmp_conf); + auto segment = NewSegment(collection, 0); int N = 10000; auto [raw_data, timestamps, uids] = generate_data(N); @@ -480,23 +426,18 @@ TEST(CApiTest, GetRowCountTest) { assert(row_count == N); DeleteCollection(collection); - DeletePartition(partition); DeleteSegment(segment); } -TEST(CApiTest, SchemaTest) { - std::string schema_string = - "id: 6873737669791618215\nname: \"collection0\"\nschema: \u003c\n " - "field_metas: \u003c\n field_name: \"age\"\n type: INT32\n dim: 1\n \u003e\n " - "field_metas: \u003c\n field_name: \"field_1\"\n type: VECTOR_FLOAT\n dim: 16\n \u003e\n" - "\u003e\ncreate_time: 1600416765\nsegment_ids: 6873737669791618215\npartition_tags: \"default\"\n"; - - auto collection_name = "collection0"; - auto collection = NewCollection(collection_name, schema_string.data()); - auto partition_name = "partition0"; - auto partition = NewPartition(collection, partition_name); - auto segment = NewSegment(partition, 0); - DeleteCollection(collection); - DeletePartition(partition); - DeleteSegment(segment); -} \ No newline at end of file +//TEST(CApiTest, SchemaTest) { +// std::string schema_string = +// "id: 6873737669791618215\nname: \"collection0\"\nschema: \u003c\n " +// "field_metas: \u003c\n field_name: \"age\"\n type: INT32\n dim: 1\n \u003e\n " +// "field_metas: \u003c\n field_name: \"field_1\"\n type: VECTOR_FLOAT\n dim: 16\n \u003e\n" +// "\u003e\ncreate_time: 1600416765\nsegment_ids: 6873737669791618215\npartition_tags: \"default\"\n"; +// +// auto collection = NewCollection(schema_string.data()); +// auto segment = NewSegment(collection, 0); +// DeleteCollection(collection); +// DeleteSegment(segment); +//} \ No newline at end of file diff --git a/internal/reader/col_seg_container.go b/internal/reader/col_seg_container.go index 3c79965b5a..bfb0f05b06 100644 --- a/internal/reader/col_seg_container.go +++ b/internal/reader/col_seg_container.go @@ -7,7 +7,6 @@ package reader #cgo LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_segcore -Wl,-rpath=${SRCDIR}/../core/output/lib #include "collection_c.h" -#include "partition_c.h" #include "segment_c.h" */ diff --git a/internal/reader/collection.go b/internal/reader/collection.go index ad76991379..98039d84ae 100644 --- a/internal/reader/collection.go +++ b/internal/reader/collection.go @@ -7,7 +7,6 @@ package reader #cgo LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_segcore -Wl,-rpath=${SRCDIR}/../core/output/lib #include "collection_c.h" -#include "partition_c.h" #include "segment_c.h" */ @@ -39,9 +38,8 @@ func newCollection(collMeta *etcdpb.CollectionMeta, collMetaBlob string) *Collec CCollection newCollection(const char* schema_conf); */ - cTmp := C.CString("tmp") cCollMetaBlob := C.CString(collMetaBlob) - collection := C.NewCollection(cTmp, cCollMetaBlob) + collection := C.NewCollection(cCollMetaBlob) var newCollection = &Collection{collectionPtr: collection, meta: collMeta} diff --git a/internal/reader/fg_delete_node.go b/internal/reader/flow_graph_delete_node.go similarity index 100% rename from internal/reader/fg_delete_node.go rename to internal/reader/flow_graph_delete_node.go diff --git a/internal/reader/fg_filter_dm_node.go b/internal/reader/flow_graph_filter_dm_node.go similarity index 100% rename from internal/reader/fg_filter_dm_node.go rename to internal/reader/flow_graph_filter_dm_node.go diff --git a/internal/reader/fg_insert_node.go b/internal/reader/flow_graph_insert_node.go similarity index 100% rename from internal/reader/fg_insert_node.go rename to internal/reader/flow_graph_insert_node.go diff --git a/internal/reader/fg_key2seg_node.go b/internal/reader/flow_graph_key2seg_node.go similarity index 100% rename from internal/reader/fg_key2seg_node.go rename to internal/reader/flow_graph_key2seg_node.go diff --git a/internal/reader/fg_message.go b/internal/reader/flow_graph_message.go similarity index 100% rename from internal/reader/fg_message.go rename to internal/reader/flow_graph_message.go diff --git a/internal/reader/fg_msg_stream_input_nodes.go b/internal/reader/flow_graph_msg_stream_input_nodes.go similarity index 100% rename from internal/reader/fg_msg_stream_input_nodes.go rename to internal/reader/flow_graph_msg_stream_input_nodes.go diff --git a/internal/reader/fg_node.go b/internal/reader/flow_graph_node.go similarity index 100% rename from internal/reader/fg_node.go rename to internal/reader/flow_graph_node.go diff --git a/internal/reader/fg_schema_update_node.go b/internal/reader/flow_graph_schema_update_node.go similarity index 100% rename from internal/reader/fg_schema_update_node.go rename to internal/reader/flow_graph_schema_update_node.go diff --git a/internal/reader/fg_service_time_node.go b/internal/reader/flow_graph_service_time_node.go similarity index 100% rename from internal/reader/fg_service_time_node.go rename to internal/reader/flow_graph_service_time_node.go diff --git a/internal/reader/index.go b/internal/reader/index.go index 1b4f2e68d7..f30979d876 100644 --- a/internal/reader/index.go +++ b/internal/reader/index.go @@ -7,7 +7,6 @@ package reader #cgo LDFLAGS: -L../core/output/lib -lmilvus_segcore -Wl,-rpath=../core/output/lib #include "collection_c.h" -#include "partition_c.h" #include "segment_c.h" */ diff --git a/internal/reader/partition.go b/internal/reader/partition.go index 323426d32d..ff4b6dcd01 100644 --- a/internal/reader/partition.go +++ b/internal/reader/partition.go @@ -7,7 +7,6 @@ package reader #cgo LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_segcore -Wl,-rpath=${SRCDIR}/../core/output/lib #include "collection_c.h" -#include "partition_c.h" #include "segment_c.h" */ diff --git a/internal/reader/query_node.go b/internal/reader/query_node.go index a1add72c0d..427fd12f0a 100644 --- a/internal/reader/query_node.go +++ b/internal/reader/query_node.go @@ -7,7 +7,6 @@ package reader #cgo LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_segcore -Wl,-rpath=${SRCDIR}/../core/output/lib #include "collection_c.h" -#include "partition_c.h" #include "segment_c.h" */ diff --git a/internal/reader/segment.go b/internal/reader/segment.go index 214c512334..7c67fac8fb 100644 --- a/internal/reader/segment.go +++ b/internal/reader/segment.go @@ -7,7 +7,6 @@ package reader #cgo LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_segcore -Wl,-rpath=${SRCDIR}/../core/output/lib #include "collection_c.h" -#include "partition_c.h" #include "segment_c.h" */ @@ -41,8 +40,7 @@ func newSegment(collection *Collection, segmentID int64) *Segment { CSegmentBase newSegment(CPartition partition, unsigned long segment_id); */ - var tmp C.CPartition - segmentPtr := C.NewSegment(tmp, C.ulong(segmentID)) + segmentPtr := C.NewSegment(collection.collectionPtr, C.ulong(segmentID)) var newSegment = &Segment{segmentPtr: segmentPtr, segmentID: segmentID} return newSegment