diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index b9f7fb662d..d3cfcc4b09 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -9,3 +9,12 @@ set (CMAKE_MODULE_PATH "${CMAKE_MODULE_PATH};${CMAKE_CURRENT_SOURCE_DIR}/cmake") include_directories(src) add_subdirectory(src) add_subdirectory(unittest) + +install( + DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/src/dog_segment/ + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/include + FILES_MATCHING PATTERN "*_c.h" +) + +install(FILES ${CMAKE_BINARY_DIR}/src/dog_segment/libmilvus_dog_segment.so + DESTINATION ${CMAKE_CURRENT_SOURCE_DIR}/lib) diff --git a/core/build.sh b/core/build.sh new file mode 100755 index 0000000000..376c65efe1 --- /dev/null +++ b/core/build.sh @@ -0,0 +1,8 @@ +if [[ -d "./build" ]] +then + rm -rf build +fi + +mkdir build && cd build +cmake .. +make -j8 && make install \ No newline at end of file diff --git a/core/include/collection_c.h b/core/include/collection_c.h new file mode 100644 index 0000000000..e69de29bb2 diff --git a/core/src/dog_segment/cwrap.h b/core/include/segment_c.h similarity index 92% rename from core/src/dog_segment/cwrap.h rename to core/include/segment_c.h index 63ed8eb8fb..1be07d4ff6 100644 --- a/core/src/dog_segment/cwrap.h +++ b/core/include/segment_c.h @@ -10,7 +10,7 @@ extern "C" { typedef void* CSegmentBase; -CSegmentBase SegmentBaseInit(); +CSegmentBase SegmentBaseInit(unsigned long segment_id); //int32_t Insert(CSegmentBase c_segment, signed long int size, const unsigned long* primary_keys, const unsigned long int* timestamps, DogDataChunk values); diff --git a/core/src/dog_segment/CMakeLists.txt b/core/src/dog_segment/CMakeLists.txt index 7205a0d808..41b0f4be21 100644 --- a/core/src/dog_segment/CMakeLists.txt +++ b/core/src/dog_segment/CMakeLists.txt @@ -1,7 +1,10 @@ set(DOG_SEGMENT_FILES SegmentNaive.cpp Collection.cpp - cwrap.cpp + Partition.cpp + collection_c.cpp + partition_c.cpp + segment_c.cpp ) # Third Party dablooms file #aux_source_directory( ${MILVUS_THIRDPARTY_SRC}/dablooms THIRDPARTY_DABLOOMS_FILES ) @@ -11,7 +14,3 @@ add_library(milvus_dog_segment SHARED #add_dependencies( segment sqlite mysqlpp ) target_link_libraries(milvus_dog_segment tbb milvus_utils pthread) - -#add_executable(main main.cpp) -#target_link_libraries(main -# milvus_dog_segment) diff --git a/core/src/dog_segment/Collection.cpp b/core/src/dog_segment/Collection.cpp index 0373659df1..b45b82d60d 100644 --- a/core/src/dog_segment/Collection.cpp +++ b/core/src/dog_segment/Collection.cpp @@ -1,2 +1,17 @@ #include "Collection.h" +namespace milvus::dog_segment { + +Collection::Collection(std::string &collection_name, std::string &schema): + collection_name_(collection_name), schema_json_(schema){} + +void +Collection::set_index() {} + +void +Collection::parse() {} + +void +Collection::AddNewPartition() {} + +} diff --git a/core/src/dog_segment/Collection.h b/core/src/dog_segment/Collection.h index 44cf7e8f4b..2ad0b47634 100644 --- a/core/src/dog_segment/Collection.h +++ b/core/src/dog_segment/Collection.h @@ -1,52 +1,21 @@ #pragma once +#include "dog_segment/Partition.h" #include "SegmentDefs.h" -#include "SegmentBase.h" - -////////////////////////////////////////////////////////////////// namespace milvus::dog_segment { -class Partition { -public: - explicit Partition(std::string& partition_name): partition_name_(partition_name) {} - - const std::vector &segments() const { - return segments_; - } - -private: - std::string partition_name_; - std::vector segments_; -}; - -using PartitionPtr = std::shared_ptr; - -////////////////////////////////////////////////////////////////// - class Collection { public: - explicit Collection(std::string &collection_name, std::string &schema) - : collection_name_(collection_name), schema_json_(schema) {} + explicit Collection(std::string &collection_name, std::string &schema); // TODO: set index - void set_index() {} + void set_index(); - void parse() { - // TODO: config to schema - } + // TODO: config to schema + void parse(); -public: - -// std::vector Insert() { -// for (auto partition: partitions_) { -// for (auto segment: partition.segments()) { -// if (segment.Status == Status.open) { -// segment.Insert() -// } -// } -// } -// } + void AddNewPartition(); private: // TODO: add Index ptr diff --git a/core/src/dog_segment/Partition.cpp b/core/src/dog_segment/Partition.cpp new file mode 100644 index 0000000000..cd31a5ca50 --- /dev/null +++ b/core/src/dog_segment/Partition.cpp @@ -0,0 +1,20 @@ +#include "Partition.h" + +namespace milvus::dog_segment { + +Partition::Partition(std::string& partition_name): + partition_name_(partition_name) {} + +void +Partition::AddNewSegment(uint64_t segment_id) { + auto segment = CreateSegment(); + segment->set_segment_id(segment_id); + segments_.emplace_back(segment); +} + +Partition* +CreatePartition() { + +} + +} diff --git a/core/src/dog_segment/Partition.h b/core/src/dog_segment/Partition.h new file mode 100644 index 0000000000..bea4f26276 --- /dev/null +++ b/core/src/dog_segment/Partition.h @@ -0,0 +1,26 @@ +#pragma once + +#include "SegmentBase.h" + +namespace milvus::dog_segment { + +class Partition { +public: + explicit Partition(std::string& partition_name); + + const std::vector &segments() const { + return segments_; + } + + void AddNewSegment(uint64_t segment_id); + +private: + std::string partition_name_; + std::vector segments_; +}; + +using PartitionPtr = std::shared_ptr; + +Partition* CreatePartiton(); + +} \ No newline at end of file diff --git a/core/src/dog_segment/SegmentNaive.cpp b/core/src/dog_segment/SegmentNaive.cpp index 3621022b7c..08821d03b6 100644 --- a/core/src/dog_segment/SegmentNaive.cpp +++ b/core/src/dog_segment/SegmentNaive.cpp @@ -171,7 +171,6 @@ SegmentNaive::Insert(int64_t size, const uint64_t* primary_keys, const Timestamp const auto& schema = *schema_; auto data_chunk = ColumnBasedDataChunk::from(row_values, schema); - std::cout << "key:" << std::endl; // insert datas // TODO: use shared_lock std::lock_guard lck(mutex_); @@ -180,7 +179,6 @@ SegmentNaive::Insert(int64_t size, const uint64_t* primary_keys, const Timestamp uids_.grow_by(primary_keys, primary_keys + size); for(int64_t i = 0; i < size; ++i) { auto key = primary_keys[i]; - std::cout << key << std::endl; auto internal_index = i + ack_id; internal_indexes_[key] = internal_index; } diff --git a/core/src/dog_segment/collection_c.cpp b/core/src/dog_segment/collection_c.cpp new file mode 100644 index 0000000000..4ba3fc5618 --- /dev/null +++ b/core/src/dog_segment/collection_c.cpp @@ -0,0 +1,11 @@ +#include "collection_c.h" + +CCollection +NewCollection(const char* collection_name) { + +} + +void +DeleteCollection(CCollection collection) { + +} diff --git a/core/src/dog_segment/collection_c.h b/core/src/dog_segment/collection_c.h new file mode 100644 index 0000000000..fd4a96f118 --- /dev/null +++ b/core/src/dog_segment/collection_c.h @@ -0,0 +1,13 @@ +#ifdef __cplusplus +extern "C" { +#endif + +typedef void* CCollection; + +CCollection NewCollection(const char* collection_name); + +void DeleteCollection(CCollection collection); + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/core/src/dog_segment/main.cpp b/core/src/dog_segment/main.cpp deleted file mode 100644 index 5781a3c6d4..0000000000 --- a/core/src/dog_segment/main.cpp +++ /dev/null @@ -1,67 +0,0 @@ -#include -#include -#include -#include -#include "cwrap.h" - -//int main() { -// auto s = SegmentBaseInit(); -// -// std::vector raw_data; -// std::vector timestamps; -// std::vector uids; -// int N = 10000; -// std::default_random_engine e(67); -// for(int i = 0; i < N; ++i) { -// uids.push_back(100000 + i); -// timestamps.push_back(0); -// // append vec -// float vec[16]; -// for(auto &x: vec) { -// x = e() % 2000 * 0.001 - 1.0; -// } -// raw_data.insert(raw_data.end(), (const char*)std::begin(vec), (const char*)std::end(vec)); -// int age = e() % 100; -// raw_data.insert(raw_data.end(), (const char*)&age, ((const char*)&age) + sizeof(age)); -// } -// -// auto line_sizeof = (sizeof(int) + sizeof(float) * 16); -// -// DogDataChunk dogDataChunk{}; -// dogDataChunk.count = N; -// dogDataChunk.raw_data = raw_data.data(); -// dogDataChunk.sizeof_per_row = (int)line_sizeof; -// -// auto res = Insert(s, N, uids.data(), timestamps.data(), dogDataChunk); -// -// std::cout << res << std::endl; -//} - -int main() { - auto s = SegmentBaseInit(); - - std::vector raw_data; - std::vector timestamps; - std::vector uids; - int N = 10000; - std::default_random_engine e(67); - for(int i = 0; i < N; ++i) { - uids.push_back(100000 + i); - timestamps.push_back(0); - // append vec - float vec[16]; - for(auto &x: vec) { - x = e() % 2000 * 0.001 - 1.0; - } - raw_data.insert(raw_data.end(), (const char*)std::begin(vec), (const char*)std::end(vec)); - int age = e() % 100; - raw_data.insert(raw_data.end(), (const char*)&age, ((const char*)&age) + sizeof(age)); - } - - auto line_sizeof = (sizeof(int) + sizeof(float) * 16); - - auto res = Insert(s, N, uids.data(), timestamps.data(), raw_data.data(), (int)line_sizeof, N); - - std::cout << res << std::endl; -} - diff --git a/core/src/dog_segment/partition_c.cpp b/core/src/dog_segment/partition_c.cpp new file mode 100644 index 0000000000..a40b4e30eb --- /dev/null +++ b/core/src/dog_segment/partition_c.cpp @@ -0,0 +1,12 @@ +#include "partition_c.h" +#include "Partition.h" +#include "Collection.h" + +CPartition +NewPartition(CCollection collection, const char* partition_name) { + auto name = std::string(partition_name); + auto partition = new milvus::dog_segment::Partition(name); + + auto co = (milvus::dog_segment::Collection*)collection; + co->AddNewPartition(); +} diff --git a/core/src/dog_segment/partition_c.h b/core/src/dog_segment/partition_c.h new file mode 100644 index 0000000000..96e86cac72 --- /dev/null +++ b/core/src/dog_segment/partition_c.h @@ -0,0 +1,15 @@ +#ifdef __cplusplus +extern "C" { +#endif + +#include "collection_c.h" + +typedef void* CPartition; + +CPartition NewPartition(CCollection collection, const char* partition_name); + +void DeletePartition(CPartition partition); + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/core/src/dog_segment/cwrap.cpp b/core/src/dog_segment/segment_c.cpp similarity index 92% rename from core/src/dog_segment/cwrap.cpp rename to core/src/dog_segment/segment_c.cpp index 654816b21e..f337c4babd 100644 --- a/core/src/dog_segment/cwrap.cpp +++ b/core/src/dog_segment/segment_c.cpp @@ -1,10 +1,11 @@ #include "SegmentBase.h" -#include "cwrap.h" +#include "segment_c.h" CSegmentBase -SegmentBaseInit() { +SegmentBaseInit(unsigned long segment_id) { std::cout << "Hello milvus" << std::endl; auto seg = milvus::dog_segment::CreateSegment(); + seg->set_segment_id(segment_id); return (void*)seg; } diff --git a/core/src/dog_segment/segment_c.h b/core/src/dog_segment/segment_c.h new file mode 100644 index 0000000000..1be07d4ff6 --- /dev/null +++ b/core/src/dog_segment/segment_c.h @@ -0,0 +1,27 @@ +#ifdef __cplusplus +extern "C" { +#endif + +//struct DogDataChunk { +// void* raw_data; // schema +// int sizeof_per_row; // alignment +// signed long int count; +//}; + +typedef void* CSegmentBase; + +CSegmentBase SegmentBaseInit(unsigned long segment_id); + +//int32_t Insert(CSegmentBase c_segment, signed long int size, const unsigned long* primary_keys, const unsigned long int* timestamps, DogDataChunk values); + +int Insert(CSegmentBase c_segment, + signed long int size, + const unsigned long* primary_keys, + const unsigned long int* timestamps, + void* raw_data, + int sizeof_per_row, + signed long int count); + +#ifdef __cplusplus +} +#endif \ No newline at end of file diff --git a/core/unittest/CMakeLists.txt b/core/unittest/CMakeLists.txt index 5c07c0d9b4..79b707dc65 100644 --- a/core/unittest/CMakeLists.txt +++ b/core/unittest/CMakeLists.txt @@ -2,6 +2,7 @@ enable_testing() find_package(GTest REQUIRED) set(MILVUS_TEST_FILES test_dog_segment.cpp + test_c_api.cpp ) add_executable(all_tests ${MILVUS_TEST_FILES} diff --git a/core/unittest/test_c_api.cpp b/core/unittest/test_c_api.cpp new file mode 100644 index 0000000000..f2843182c5 --- /dev/null +++ b/core/unittest/test_c_api.cpp @@ -0,0 +1,36 @@ +#include +#include +#include +#include + +#include "dog_segment/segment_c.h" +#include "dog_segment/collection_c.h" + +TEST(SegmentTest, InsertTest) { + auto segment_id = 0; + auto s = SegmentBaseInit(segment_id); + + std::vector raw_data; + std::vector timestamps; + std::vector uids; + int N = 10000; + std::default_random_engine e(67); + for(int i = 0; i < N; ++i) { + uids.push_back(100000 + i); + timestamps.push_back(0); + // append vec + float vec[16]; + for(auto &x: vec) { + x = e() % 2000 * 0.001 - 1.0; + } + raw_data.insert(raw_data.end(), (const char*)std::begin(vec), (const char*)std::end(vec)); + int age = e() % 100; + raw_data.insert(raw_data.end(), (const char*)&age, ((const char*)&age) + sizeof(age)); + } + + auto line_sizeof = (sizeof(int) + sizeof(float) * 16); + + auto res = Insert(s, N, uids.data(), timestamps.data(), raw_data.data(), (int)line_sizeof, N); + + std::cout << res << std::endl; +} \ No newline at end of file diff --git a/reader/query_node.go b/reader/query_node.go index 5b985dd78e..dc6a8b2265 100644 --- a/reader/query_node.go +++ b/reader/query_node.go @@ -119,7 +119,8 @@ func (node *QueryNode) InitQueryNodeCollection() { node.Collections = append(node.Collections, collection) var partition, _ = collection.NewPartition("partition1") collection.Partitions = append(collection.Partitions, partition) - var segment, _ = partition.NewSegment() + // TODO: add segment id + var segment, _ = partition.NewSegment(0) partition.Segments = append(partition.Segments, segment) } @@ -128,10 +129,11 @@ func (node *QueryNode) SegmentsManagement() { for _, collection := range node.Collections { for _, partition := range collection.Partitions { for _, segment := range partition.Segments { + // TODO: check segment status if timeSync >= segment.SegmentCloseTime { segment.Close() // TODO: add atomic segment id - var newSegment, _ = partition.NewSegment() + var newSegment, _ = partition.NewSegment(0) newSegment.SegmentCloseTime = timeSync + SegmentLifetime partition.Segments = append(partition.Segments, newSegment) } diff --git a/reader/segment.go b/reader/segment.go index 66ffc9e877..45dcd6cceb 100644 --- a/reader/segment.go +++ b/reader/segment.go @@ -1,5 +1,14 @@ package reader +/* + +#cgo CFLAGS: -I../core/include + +#cgo LDFLAGS: -L../core/lib -lmilvus_dog_segment -Wl,-rpath=../core/lib + +#include "segment_c.h" + +*/ import "C" import ( "errors" @@ -14,9 +23,9 @@ type Segment struct { SegmentCloseTime uint64 } -func (p *Partition) NewSegment() (*Segment, error) { +func (p *Partition) NewSegment(segmentId uint64) (*Segment, error) { // TODO: add segment id - segmentPtr, status := C.CreateSegment(p.PartitionPtr) + segmentPtr, status := C.SegmentBaseInit(p.PartitionPtr) if status != 0 { return nil, errors.New("create segment failed") diff --git a/reader/test/cgo_segment_test.go b/reader/test/cgo_segment_test.go new file mode 100644 index 0000000000..a81b934389 --- /dev/null +++ b/reader/test/cgo_segment_test.go @@ -0,0 +1,10 @@ +package main + +import ( + "fmt" + "testing" +) + +func TestIntMinBasic(t *testing.T) { + fmt.Println("Hello go testing") +} \ No newline at end of file