admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor index wrapper, use protobuf instead of json to pass data

Browse Source 
Signed-off-by: dragondriver <jiquan.long@zilliz.com>
This commit is contained in:
dragondriver 2020-12-21 16:36:07 +08:00 committed by yefu.chen
parent 87521adfbd
commit d023c1afba
11 changed files with 2582 additions and 43 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

77
internal/core/src/indexbuilder/IndexWrapper.cpp
View File

@ -9,16 +9,19 @@
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#include "pb/index_cgo_msg.pb.h"
#include "knowhere/index/vector_index/VecIndexFactory.h"
#include "knowhere/index/vector_index/helpers/IndexParameter.h"
#include "utils/EasyAssert.h"
#include "IndexWrapper.h"
namespace milvus {
namespace indexbuilder {
IndexWrapper::IndexWrapper(const char* type_params_str, const char* index_params_str) {
type_params_ = std::string(type_params_str);
index_params_ = std::string(index_params_str);
IndexWrapper::IndexWrapper(const char* serialized_type_params, const char* serialized_index_params) {
type_params_ = std::string(serialized_type_params);
index_params_ = std::string(serialized_index_params);
parse();
auto index_type = index_config_["index_type"].get<std::string>();
@ -30,8 +33,26 @@ IndexWrapper::IndexWrapper(const char* type_params_str, const char* index_params
void
IndexWrapper::parse() {
type_config_ = milvus::json::parse(type_params_);
index_config_ = milvus::json::parse(index_params_);
namespace indexcgo = milvus::proto::indexcgo;
bool serialized_success;
indexcgo::BinarySet type_config;
serialized_success = type_config.ParseFromString(type_params_);
Assert(serialized_success);
indexcgo::BinarySet index_config;
serialized_success = index_config.ParseFromString(index_params_);
Assert(serialized_success);
for (auto i = 0; i < type_config.datas_size(); ++i) {
auto binary = type_config.datas(i);
type_config_[binary.key()] = binary.value();
}
for (auto i = 0; i < index_config.datas_size(); ++i) {
auto binary = index_config.datas(i);
index_config_[binary.key()] = binary.value();
}
// TODO: parse from type_params & index_params
auto dim = 128;
@ -55,14 +76,52 @@ IndexWrapper::BuildWithoutIds(const knowhere::DatasetPtr& dataset) {
index_->AddWithoutIds(dataset, config_);
}
char*
/*
* brief Return serialized binary set
*/
milvus::indexbuilder::IndexWrapper::Binary
IndexWrapper::Serialize() {
return (char*)malloc(1);
namespace indexcgo = milvus::proto::indexcgo;
auto binarySet = index_->Serialize(config_);
indexcgo::BinarySet ret;
for (auto [key, value] : binarySet.binary_map_) {
auto binary = ret.add_datas();
binary->set_key(key);
binary->set_value(reinterpret_cast<char*>(value->data.get()));
}
std::string serialized_data;
auto ok = ret.SerializeToString(&serialized_data);
Assert(ok);
auto data = new char[serialized_data.length() + 1];
memcpy(data, serialized_data.c_str(), serialized_data.length());
data[serialized_data.length()] = 0;
return {data, static_cast<int32_t>(serialized_data.length() + 1)};
}
void
IndexWrapper::Load(const char* dumped_blob_buffer) {
return;
IndexWrapper::Load(const char* serialized_sliced_blob_buffer) {
namespace indexcgo = milvus::proto::indexcgo;
auto data = std::string(serialized_sliced_blob_buffer);
indexcgo::BinarySet blob_buffer;
auto ok = blob_buffer.ParseFromString(data);
Assert(ok);
milvus::knowhere::BinarySet binarySet;
for (auto i = 0; i < blob_buffer.datas_size(); i++) {
auto binary = blob_buffer.datas(i);
std::shared_ptr<uint8_t[]> binary_data(new uint8_t[binary.value().length() + 1],
std::default_delete<uint8_t[]>());
memcpy(binary_data.get(), binary.value().c_str(), binary.value().length());
binary_data[binary.value().length()] = 0;
binarySet.Append(binary.key(), binary_data, binary.value().length() + 1);
}
index_->Load(binarySet);
}
} // namespace indexbuilder

11
internal/core/src/indexbuilder/IndexWrapper.h
View File

@ -17,7 +17,7 @@ namespace indexbuilder {
class IndexWrapper {
public:
explicit IndexWrapper(const char* type_params_str, const char* index_params_str);
explicit IndexWrapper(const char* serialized_type_params, const char* serialized_index_params);
int64_t
dim();
@ -25,11 +25,16 @@ class IndexWrapper {
void
BuildWithoutIds(const knowhere::DatasetPtr& dataset);
char*
struct Binary {
char* data;
int32_t size;
};
Binary
Serialize();
void
Load(const char* dumped_blob_buffer);
Load(const char* serialized_sliced_blob_buffer);
private:
void

17
internal/core/src/indexbuilder/index_c.cpp
View File

@ -15,8 +15,8 @@
#include "indexbuilder/index_c.h"
CIndex
CreateIndex(const char* type_params_str, const char* index_params_str) {
auto index = std::make_unique<milvus::indexbuilder::IndexWrapper>(type_params_str, index_params_str);
CreateIndex(const char* serialized_type_params, const char* serialized_index_params) {
auto index = std::make_unique<milvus::indexbuilder::IndexWrapper>(serialized_type_params, serialized_index_params);
return (void*)(index.release());
}
@ -28,21 +28,24 @@ DeleteIndex(CIndex index) {
}
void
BuildFloatVecIndex(CIndex index, int64_t row_nums, const float* vectors) {
BuildFloatVecIndex(CIndex index, int64_t float_value_num, const float* vectors) {
auto cIndex = (milvus::indexbuilder::IndexWrapper*)index;
auto dim = cIndex->dim();
auto row_nums = float_value_num / dim;
auto ds = milvus::knowhere::GenDataset(row_nums, dim, vectors);
cIndex->BuildWithoutIds(ds);
}
char*
SerializeToSlicedBuffer(CIndex index) {
SerializeToSlicedBuffer(CIndex index, int32_t* buffer_size) {
auto cIndex = (milvus::indexbuilder::IndexWrapper*)index;
return cIndex->Serialize();
auto binary = cIndex->Serialize();
*buffer_size = binary.size;
return binary.data;
}
void
LoadFromSlicedBuffer(CIndex index, const char* dumped_blob_buffer) {
LoadFromSlicedBuffer(CIndex index, const char* serialized_sliced_blob_buffer) {
auto cIndex = (milvus::indexbuilder::IndexWrapper*)index;
cIndex->Load(dumped_blob_buffer);
cIndex->Load(serialized_sliced_blob_buffer);
}

10
internal/core/src/indexbuilder/index_c.h
View File

@ -32,20 +32,22 @@ extern "C" {
typedef void* CIndex;
// TODO: how could we pass map between go and c++ more efficiently?
// Solution: using protobuf instead of json, this way significantly increase programming efficiency
CIndex
CreateIndex(const char* type_params_str, const char* index_params_str);
CreateIndex(const char* serialized_type_params, const char* serialized_index_params);
void
DeleteIndex(CIndex index);
void
BuildFloatVecIndex(CIndex index, const float* vectors);
BuildFloatVecIndex(CIndex index, int64_t float_value_num, const float* vectors);
char*
SerializeToSlicedBuffer(CIndex index);
SerializeToSlicedBuffer(CIndex index, int32_t* buffer_size);
void
LoadFromSlicedBuffer(CIndex index, const char* dumped_blob_buffer);
LoadFromSlicedBuffer(CIndex index, const char* serialized_sliced_blob_buffer);
#ifdef __cplusplus
};

2
internal/core/src/pb/CMakeLists.txt
View File

@ -1,7 +1,7 @@
get_property(PROTOC_EXCUTABLE GLOBAL PROPERTY PROTOC_EXCUTABLE )
set(proto_file_names common.proto etcd_meta.proto schema.proto service_msg.proto)
set(proto_file_names common.proto etcd_meta.proto schema.proto service_msg.proto index_cgo_msg.proto)
set( PROTO_PATH "${MILVUS_SOURCE_DIR}/../proto/" )

1311
internal/core/src/pb/index_cgo_msg.pb.cc Normal file
View File

File diff suppressed because it is too large Load Diff

875
internal/core/src/pb/index_cgo_msg.pb.h Normal file
View File

@ -0,0 +1,875 @@
// Generated by the protocol buffer compiler. DO NOT EDIT!
// source: index_cgo_msg.proto
#ifndef GOOGLE_PROTOBUF_INCLUDED_index_5fcgo_5fmsg_2eproto
#define GOOGLE_PROTOBUF_INCLUDED_index_5fcgo_5fmsg_2eproto
#include <limits>
#include <string>
#include <google/protobuf/port_def.inc>
#if PROTOBUF_VERSION < 3009000
#error This file was generated by a newer version of protoc which is
#error incompatible with your Protocol Buffer headers. Please update
#error your headers.
#endif
#if 3009000 < PROTOBUF_MIN_PROTOC_VERSION
#error This file was generated by an older version of protoc which is
#error incompatible with your Protocol Buffer headers. Please
#error regenerate this file with a newer version of protoc.
#endif
#include <google/protobuf/port_undef.inc>
#include <google/protobuf/io/coded_stream.h>
#include <google/protobuf/arena.h>
#include <google/protobuf/arenastring.h>
#include <google/protobuf/generated_message_table_driven.h>
#include <google/protobuf/generated_message_util.h>
#include <google/protobuf/inlined_string_field.h>
#include <google/protobuf/metadata.h>
#include <google/protobuf/generated_message_reflection.h>
#include <google/protobuf/message.h>
#include <google/protobuf/repeated_field.h> // IWYU pragma: export
#include <google/protobuf/extension_set.h> // IWYU pragma: export
#include <google/protobuf/unknown_field_set.h>
#include "common.pb.h"
// @@protoc_insertion_point(includes)
#include <google/protobuf/port_def.inc>
#define PROTOBUF_INTERNAL_EXPORT_index_5fcgo_5fmsg_2eproto
PROTOBUF_NAMESPACE_OPEN
namespace internal {
class AnyMetadata;
} // namespace internal
PROTOBUF_NAMESPACE_CLOSE
// Internal implementation detail -- do not use these members.
struct TableStruct_index_5fcgo_5fmsg_2eproto {
static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTableField entries[]
PROTOBUF_SECTION_VARIABLE(protodesc_cold);
static const ::PROTOBUF_NAMESPACE_ID::internal::AuxillaryParseTableField aux[]
PROTOBUF_SECTION_VARIABLE(protodesc_cold);
static const ::PROTOBUF_NAMESPACE_ID::internal::ParseTable schema[4]
PROTOBUF_SECTION_VARIABLE(protodesc_cold);
static const ::PROTOBUF_NAMESPACE_ID::internal::FieldMetadata field_metadata[];
static const ::PROTOBUF_NAMESPACE_ID::internal::SerializationTable serialization_table[];
static const ::PROTOBUF_NAMESPACE_ID::uint32 offsets[];
};
extern const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_index_5fcgo_5fmsg_2eproto;
namespace milvus {
namespace proto {
namespace indexcgo {
class Binary;
class BinaryDefaultTypeInternal;
extern BinaryDefaultTypeInternal _Binary_default_instance_;
class BinarySet;
class BinarySetDefaultTypeInternal;
extern BinarySetDefaultTypeInternal _BinarySet_default_instance_;
class IndexParams;
class IndexParamsDefaultTypeInternal;
extern IndexParamsDefaultTypeInternal _IndexParams_default_instance_;
class TypeParams;
class TypeParamsDefaultTypeInternal;
extern TypeParamsDefaultTypeInternal _TypeParams_default_instance_;
} // namespace indexcgo
} // namespace proto
} // namespace milvus
PROTOBUF_NAMESPACE_OPEN
template<> ::milvus::proto::indexcgo::Binary* Arena::CreateMaybeMessage<::milvus::proto::indexcgo::Binary>(Arena*);
template<> ::milvus::proto::indexcgo::BinarySet* Arena::CreateMaybeMessage<::milvus::proto::indexcgo::BinarySet>(Arena*);
template<> ::milvus::proto::indexcgo::IndexParams* Arena::CreateMaybeMessage<::milvus::proto::indexcgo::IndexParams>(Arena*);
template<> ::milvus::proto::indexcgo::TypeParams* Arena::CreateMaybeMessage<::milvus::proto::indexcgo::TypeParams>(Arena*);
PROTOBUF_NAMESPACE_CLOSE
namespace milvus {
namespace proto {
namespace indexcgo {
// ===================================================================
class TypeParams :
public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.proto.indexcgo.TypeParams) */ {
public:
TypeParams();
virtual ~TypeParams();
TypeParams(const TypeParams& from);
TypeParams(TypeParams&& from) noexcept
: TypeParams() {
*this = ::std::move(from);
}
inline TypeParams& operator=(const TypeParams& from) {
CopyFrom(from);
return *this;
}
inline TypeParams& operator=(TypeParams&& from) noexcept {
if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) {
if (this != &from) InternalSwap(&from);
} else {
CopyFrom(from);
}
return *this;
}
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() {
return GetDescriptor();
}
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() {
return GetMetadataStatic().descriptor;
}
static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() {
return GetMetadataStatic().reflection;
}
static const TypeParams& default_instance();
static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY
static inline const TypeParams* internal_default_instance() {
return reinterpret_cast<const TypeParams*>(
&_TypeParams_default_instance_);
}
static constexpr int kIndexInFileMessages =
0;
friend void swap(TypeParams& a, TypeParams& b) {
a.Swap(&b);
}
inline void Swap(TypeParams* other) {
if (other == this) return;
InternalSwap(other);
}
// implements Message ----------------------------------------------
inline TypeParams* New() const final {
return CreateMaybeMessage<TypeParams>(nullptr);
}
TypeParams* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final {
return CreateMaybeMessage<TypeParams>(arena);
}
void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final;
void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final;
void CopyFrom(const TypeParams& from);
void MergeFrom(const TypeParams& from);
PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final;
bool IsInitialized() const final;
size_t ByteSizeLong() const final;
#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final;
#else
bool MergePartialFromCodedStream(
::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) final;
#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
void SerializeWithCachedSizes(
::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const final;
::PROTOBUF_NAMESPACE_ID::uint8* InternalSerializeWithCachedSizesToArray(
::PROTOBUF_NAMESPACE_ID::uint8* target) const final;
int GetCachedSize() const final { return _cached_size_.Get(); }
private:
inline void SharedCtor();
inline void SharedDtor();
void SetCachedSize(int size) const final;
void InternalSwap(TypeParams* other);
friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata;
static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() {
return "milvus.proto.indexcgo.TypeParams";
}
private:
inline ::PROTOBUF_NAMESPACE_ID::Arena* GetArenaNoVirtual() const {
return nullptr;
}
inline void* MaybeArenaPtr() const {
return nullptr;
}
public:
::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final;
private:
static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() {
::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_index_5fcgo_5fmsg_2eproto);
return ::descriptor_table_index_5fcgo_5fmsg_2eproto.file_level_metadata[kIndexInFileMessages];
}
public:
// nested types ----------------------------------------------------
// accessors -------------------------------------------------------
enum : int {
kParamsFieldNumber = 1,
};
// repeated .milvus.proto.common.KeyValuePair params = 1;
int params_size() const;
void clear_params();
::milvus::proto::common::KeyValuePair* mutable_params(int index);
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::KeyValuePair >*
mutable_params();
const ::milvus::proto::common::KeyValuePair& params(int index) const;
::milvus::proto::common::KeyValuePair* add_params();
const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::KeyValuePair >&
params() const;
// @@protoc_insertion_point(class_scope:milvus.proto.indexcgo.TypeParams)
private:
class _Internal;
::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::KeyValuePair > params_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
friend struct ::TableStruct_index_5fcgo_5fmsg_2eproto;
};
// -------------------------------------------------------------------
class IndexParams :
public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.proto.indexcgo.IndexParams) */ {
public:
IndexParams();
virtual ~IndexParams();
IndexParams(const IndexParams& from);
IndexParams(IndexParams&& from) noexcept
: IndexParams() {
*this = ::std::move(from);
}
inline IndexParams& operator=(const IndexParams& from) {
CopyFrom(from);
return *this;
}
inline IndexParams& operator=(IndexParams&& from) noexcept {
if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) {
if (this != &from) InternalSwap(&from);
} else {
CopyFrom(from);
}
return *this;
}
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() {
return GetDescriptor();
}
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() {
return GetMetadataStatic().descriptor;
}
static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() {
return GetMetadataStatic().reflection;
}
static const IndexParams& default_instance();
static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY
static inline const IndexParams* internal_default_instance() {
return reinterpret_cast<const IndexParams*>(
&_IndexParams_default_instance_);
}
static constexpr int kIndexInFileMessages =
1;
friend void swap(IndexParams& a, IndexParams& b) {
a.Swap(&b);
}
inline void Swap(IndexParams* other) {
if (other == this) return;
InternalSwap(other);
}
// implements Message ----------------------------------------------
inline IndexParams* New() const final {
return CreateMaybeMessage<IndexParams>(nullptr);
}
IndexParams* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final {
return CreateMaybeMessage<IndexParams>(arena);
}
void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final;
void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final;
void CopyFrom(const IndexParams& from);
void MergeFrom(const IndexParams& from);
PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final;
bool IsInitialized() const final;
size_t ByteSizeLong() const final;
#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final;
#else
bool MergePartialFromCodedStream(
::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) final;
#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
void SerializeWithCachedSizes(
::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const final;
::PROTOBUF_NAMESPACE_ID::uint8* InternalSerializeWithCachedSizesToArray(
::PROTOBUF_NAMESPACE_ID::uint8* target) const final;
int GetCachedSize() const final { return _cached_size_.Get(); }
private:
inline void SharedCtor();
inline void SharedDtor();
void SetCachedSize(int size) const final;
void InternalSwap(IndexParams* other);
friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata;
static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() {
return "milvus.proto.indexcgo.IndexParams";
}
private:
inline ::PROTOBUF_NAMESPACE_ID::Arena* GetArenaNoVirtual() const {
return nullptr;
}
inline void* MaybeArenaPtr() const {
return nullptr;
}
public:
::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final;
private:
static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() {
::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_index_5fcgo_5fmsg_2eproto);
return ::descriptor_table_index_5fcgo_5fmsg_2eproto.file_level_metadata[kIndexInFileMessages];
}
public:
// nested types ----------------------------------------------------
// accessors -------------------------------------------------------
enum : int {
kParamsFieldNumber = 1,
};
// repeated .milvus.proto.common.KeyValuePair params = 1;
int params_size() const;
void clear_params();
::milvus::proto::common::KeyValuePair* mutable_params(int index);
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::KeyValuePair >*
mutable_params();
const ::milvus::proto::common::KeyValuePair& params(int index) const;
::milvus::proto::common::KeyValuePair* add_params();
const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::KeyValuePair >&
params() const;
// @@protoc_insertion_point(class_scope:milvus.proto.indexcgo.IndexParams)
private:
class _Internal;
::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::KeyValuePair > params_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
friend struct ::TableStruct_index_5fcgo_5fmsg_2eproto;
};
// -------------------------------------------------------------------
class Binary :
public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.proto.indexcgo.Binary) */ {
public:
Binary();
virtual ~Binary();
Binary(const Binary& from);
Binary(Binary&& from) noexcept
: Binary() {
*this = ::std::move(from);
}
inline Binary& operator=(const Binary& from) {
CopyFrom(from);
return *this;
}
inline Binary& operator=(Binary&& from) noexcept {
if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) {
if (this != &from) InternalSwap(&from);
} else {
CopyFrom(from);
}
return *this;
}
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() {
return GetDescriptor();
}
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() {
return GetMetadataStatic().descriptor;
}
static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() {
return GetMetadataStatic().reflection;
}
static const Binary& default_instance();
static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY
static inline const Binary* internal_default_instance() {
return reinterpret_cast<const Binary*>(
&_Binary_default_instance_);
}
static constexpr int kIndexInFileMessages =
2;
friend void swap(Binary& a, Binary& b) {
a.Swap(&b);
}
inline void Swap(Binary* other) {
if (other == this) return;
InternalSwap(other);
}
// implements Message ----------------------------------------------
inline Binary* New() const final {
return CreateMaybeMessage<Binary>(nullptr);
}
Binary* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final {
return CreateMaybeMessage<Binary>(arena);
}
void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final;
void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final;
void CopyFrom(const Binary& from);
void MergeFrom(const Binary& from);
PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final;
bool IsInitialized() const final;
size_t ByteSizeLong() const final;
#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final;
#else
bool MergePartialFromCodedStream(
::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) final;
#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
void SerializeWithCachedSizes(
::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const final;
::PROTOBUF_NAMESPACE_ID::uint8* InternalSerializeWithCachedSizesToArray(
::PROTOBUF_NAMESPACE_ID::uint8* target) const final;
int GetCachedSize() const final { return _cached_size_.Get(); }
private:
inline void SharedCtor();
inline void SharedDtor();
void SetCachedSize(int size) const final;
void InternalSwap(Binary* other);
friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata;
static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() {
return "milvus.proto.indexcgo.Binary";
}
private:
inline ::PROTOBUF_NAMESPACE_ID::Arena* GetArenaNoVirtual() const {
return nullptr;
}
inline void* MaybeArenaPtr() const {
return nullptr;
}
public:
::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final;
private:
static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() {
::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_index_5fcgo_5fmsg_2eproto);
return ::descriptor_table_index_5fcgo_5fmsg_2eproto.file_level_metadata[kIndexInFileMessages];
}
public:
// nested types ----------------------------------------------------
// accessors -------------------------------------------------------
enum : int {
kKeyFieldNumber = 1,
kValueFieldNumber = 2,
};
// string key = 1;
void clear_key();
const std::string& key() const;
void set_key(const std::string& value);
void set_key(std::string&& value);
void set_key(const char* value);
void set_key(const char* value, size_t size);
std::string* mutable_key();
std::string* release_key();
void set_allocated_key(std::string* key);
// bytes value = 2;
void clear_value();
const std::string& value() const;
void set_value(const std::string& value);
void set_value(std::string&& value);
void set_value(const char* value);
void set_value(const void* value, size_t size);
std::string* mutable_value();
std::string* release_value();
void set_allocated_value(std::string* value);
// @@protoc_insertion_point(class_scope:milvus.proto.indexcgo.Binary)
private:
class _Internal;
::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr key_;
::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr value_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
friend struct ::TableStruct_index_5fcgo_5fmsg_2eproto;
};
// -------------------------------------------------------------------
class BinarySet :
public ::PROTOBUF_NAMESPACE_ID::Message /* @@protoc_insertion_point(class_definition:milvus.proto.indexcgo.BinarySet) */ {
public:
BinarySet();
virtual ~BinarySet();
BinarySet(const BinarySet& from);
BinarySet(BinarySet&& from) noexcept
: BinarySet() {
*this = ::std::move(from);
}
inline BinarySet& operator=(const BinarySet& from) {
CopyFrom(from);
return *this;
}
inline BinarySet& operator=(BinarySet&& from) noexcept {
if (GetArenaNoVirtual() == from.GetArenaNoVirtual()) {
if (this != &from) InternalSwap(&from);
} else {
CopyFrom(from);
}
return *this;
}
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* descriptor() {
return GetDescriptor();
}
static const ::PROTOBUF_NAMESPACE_ID::Descriptor* GetDescriptor() {
return GetMetadataStatic().descriptor;
}
static const ::PROTOBUF_NAMESPACE_ID::Reflection* GetReflection() {
return GetMetadataStatic().reflection;
}
static const BinarySet& default_instance();
static void InitAsDefaultInstance(); // FOR INTERNAL USE ONLY
static inline const BinarySet* internal_default_instance() {
return reinterpret_cast<const BinarySet*>(
&_BinarySet_default_instance_);
}
static constexpr int kIndexInFileMessages =
3;
friend void swap(BinarySet& a, BinarySet& b) {
a.Swap(&b);
}
inline void Swap(BinarySet* other) {
if (other == this) return;
InternalSwap(other);
}
// implements Message ----------------------------------------------
inline BinarySet* New() const final {
return CreateMaybeMessage<BinarySet>(nullptr);
}
BinarySet* New(::PROTOBUF_NAMESPACE_ID::Arena* arena) const final {
return CreateMaybeMessage<BinarySet>(arena);
}
void CopyFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final;
void MergeFrom(const ::PROTOBUF_NAMESPACE_ID::Message& from) final;
void CopyFrom(const BinarySet& from);
void MergeFrom(const BinarySet& from);
PROTOBUF_ATTRIBUTE_REINITIALIZES void Clear() final;
bool IsInitialized() const final;
size_t ByteSizeLong() const final;
#if GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
const char* _InternalParse(const char* ptr, ::PROTOBUF_NAMESPACE_ID::internal::ParseContext* ctx) final;
#else
bool MergePartialFromCodedStream(
::PROTOBUF_NAMESPACE_ID::io::CodedInputStream* input) final;
#endif // GOOGLE_PROTOBUF_ENABLE_EXPERIMENTAL_PARSER
void SerializeWithCachedSizes(
::PROTOBUF_NAMESPACE_ID::io::CodedOutputStream* output) const final;
::PROTOBUF_NAMESPACE_ID::uint8* InternalSerializeWithCachedSizesToArray(
::PROTOBUF_NAMESPACE_ID::uint8* target) const final;
int GetCachedSize() const final { return _cached_size_.Get(); }
private:
inline void SharedCtor();
inline void SharedDtor();
void SetCachedSize(int size) const final;
void InternalSwap(BinarySet* other);
friend class ::PROTOBUF_NAMESPACE_ID::internal::AnyMetadata;
static ::PROTOBUF_NAMESPACE_ID::StringPiece FullMessageName() {
return "milvus.proto.indexcgo.BinarySet";
}
private:
inline ::PROTOBUF_NAMESPACE_ID::Arena* GetArenaNoVirtual() const {
return nullptr;
}
inline void* MaybeArenaPtr() const {
return nullptr;
}
public:
::PROTOBUF_NAMESPACE_ID::Metadata GetMetadata() const final;
private:
static ::PROTOBUF_NAMESPACE_ID::Metadata GetMetadataStatic() {
::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&::descriptor_table_index_5fcgo_5fmsg_2eproto);
return ::descriptor_table_index_5fcgo_5fmsg_2eproto.file_level_metadata[kIndexInFileMessages];
}
public:
// nested types ----------------------------------------------------
// accessors -------------------------------------------------------
enum : int {
kDatasFieldNumber = 1,
};
// repeated .milvus.proto.indexcgo.Binary datas = 1;
int datas_size() const;
void clear_datas();
::milvus::proto::indexcgo::Binary* mutable_datas(int index);
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::indexcgo::Binary >*
mutable_datas();
const ::milvus::proto::indexcgo::Binary& datas(int index) const;
::milvus::proto::indexcgo::Binary* add_datas();
const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::indexcgo::Binary >&
datas() const;
// @@protoc_insertion_point(class_scope:milvus.proto.indexcgo.BinarySet)
private:
class _Internal;
::PROTOBUF_NAMESPACE_ID::internal::InternalMetadataWithArena _internal_metadata_;
::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::indexcgo::Binary > datas_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
friend struct ::TableStruct_index_5fcgo_5fmsg_2eproto;
};
// ===================================================================
// ===================================================================
#ifdef __GNUC__
#pragma GCC diagnostic push
#pragma GCC diagnostic ignored "-Wstrict-aliasing"
#endif // __GNUC__
// TypeParams
// repeated .milvus.proto.common.KeyValuePair params = 1;
inline int TypeParams::params_size() const {
return params_.size();
}
inline ::milvus::proto::common::KeyValuePair* TypeParams::mutable_params(int index) {
// @@protoc_insertion_point(field_mutable:milvus.proto.indexcgo.TypeParams.params)
return params_.Mutable(index);
}
inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::KeyValuePair >*
TypeParams::mutable_params() {
// @@protoc_insertion_point(field_mutable_list:milvus.proto.indexcgo.TypeParams.params)
return &params_;
}
inline const ::milvus::proto::common::KeyValuePair& TypeParams::params(int index) const {
// @@protoc_insertion_point(field_get:milvus.proto.indexcgo.TypeParams.params)
return params_.Get(index);
}
inline ::milvus::proto::common::KeyValuePair* TypeParams::add_params() {
// @@protoc_insertion_point(field_add:milvus.proto.indexcgo.TypeParams.params)
return params_.Add();
}
inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::KeyValuePair >&
TypeParams::params() const {
// @@protoc_insertion_point(field_list:milvus.proto.indexcgo.TypeParams.params)
return params_;
}
// -------------------------------------------------------------------
// IndexParams
// repeated .milvus.proto.common.KeyValuePair params = 1;
inline int IndexParams::params_size() const {
return params_.size();
}
inline ::milvus::proto::common::KeyValuePair* IndexParams::mutable_params(int index) {
// @@protoc_insertion_point(field_mutable:milvus.proto.indexcgo.IndexParams.params)
return params_.Mutable(index);
}
inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::KeyValuePair >*
IndexParams::mutable_params() {
// @@protoc_insertion_point(field_mutable_list:milvus.proto.indexcgo.IndexParams.params)
return &params_;
}
inline const ::milvus::proto::common::KeyValuePair& IndexParams::params(int index) const {
// @@protoc_insertion_point(field_get:milvus.proto.indexcgo.IndexParams.params)
return params_.Get(index);
}
inline ::milvus::proto::common::KeyValuePair* IndexParams::add_params() {
// @@protoc_insertion_point(field_add:milvus.proto.indexcgo.IndexParams.params)
return params_.Add();
}
inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::common::KeyValuePair >&
IndexParams::params() const {
// @@protoc_insertion_point(field_list:milvus.proto.indexcgo.IndexParams.params)
return params_;
}
// -------------------------------------------------------------------
// Binary
// string key = 1;
inline void Binary::clear_key() {
key_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
}
inline const std::string& Binary::key() const {
// @@protoc_insertion_point(field_get:milvus.proto.indexcgo.Binary.key)
return key_.GetNoArena();
}
inline void Binary::set_key(const std::string& value) {
key_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value);
// @@protoc_insertion_point(field_set:milvus.proto.indexcgo.Binary.key)
}
inline void Binary::set_key(std::string&& value) {
key_.SetNoArena(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value));
// @@protoc_insertion_point(field_set_rvalue:milvus.proto.indexcgo.Binary.key)
}
inline void Binary::set_key(const char* value) {
GOOGLE_DCHECK(value != nullptr);
key_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
// @@protoc_insertion_point(field_set_char:milvus.proto.indexcgo.Binary.key)
}
inline void Binary::set_key(const char* value, size_t size) {
key_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(),
::std::string(reinterpret_cast<const char*>(value), size));
// @@protoc_insertion_point(field_set_pointer:milvus.proto.indexcgo.Binary.key)
}
inline std::string* Binary::mutable_key() {
// @@protoc_insertion_point(field_mutable:milvus.proto.indexcgo.Binary.key)
return key_.MutableNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
}
inline std::string* Binary::release_key() {
// @@protoc_insertion_point(field_release:milvus.proto.indexcgo.Binary.key)
return key_.ReleaseNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
}
inline void Binary::set_allocated_key(std::string* key) {
if (key != nullptr) {
} else {
}
key_.SetAllocatedNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), key);
// @@protoc_insertion_point(field_set_allocated:milvus.proto.indexcgo.Binary.key)
}
// bytes value = 2;
inline void Binary::clear_value() {
value_.ClearToEmptyNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
}
inline const std::string& Binary::value() const {
// @@protoc_insertion_point(field_get:milvus.proto.indexcgo.Binary.value)
return value_.GetNoArena();
}
inline void Binary::set_value(const std::string& value) {
value_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value);
// @@protoc_insertion_point(field_set:milvus.proto.indexcgo.Binary.value)
}
inline void Binary::set_value(std::string&& value) {
value_.SetNoArena(
&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::move(value));
// @@protoc_insertion_point(field_set_rvalue:milvus.proto.indexcgo.Binary.value)
}
inline void Binary::set_value(const char* value) {
GOOGLE_DCHECK(value != nullptr);
value_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), ::std::string(value));
// @@protoc_insertion_point(field_set_char:milvus.proto.indexcgo.Binary.value)
}
inline void Binary::set_value(const void* value, size_t size) {
value_.SetNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(),
::std::string(reinterpret_cast<const char*>(value), size));
// @@protoc_insertion_point(field_set_pointer:milvus.proto.indexcgo.Binary.value)
}
inline std::string* Binary::mutable_value() {
// @@protoc_insertion_point(field_mutable:milvus.proto.indexcgo.Binary.value)
return value_.MutableNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
}
inline std::string* Binary::release_value() {
// @@protoc_insertion_point(field_release:milvus.proto.indexcgo.Binary.value)
return value_.ReleaseNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited());
}
inline void Binary::set_allocated_value(std::string* value) {
if (value != nullptr) {
} else {
}
value_.SetAllocatedNoArena(&::PROTOBUF_NAMESPACE_ID::internal::GetEmptyStringAlreadyInited(), value);
// @@protoc_insertion_point(field_set_allocated:milvus.proto.indexcgo.Binary.value)
}
// -------------------------------------------------------------------
// BinarySet
// repeated .milvus.proto.indexcgo.Binary datas = 1;
inline int BinarySet::datas_size() const {
return datas_.size();
}
inline void BinarySet::clear_datas() {
datas_.Clear();
}
inline ::milvus::proto::indexcgo::Binary* BinarySet::mutable_datas(int index) {
// @@protoc_insertion_point(field_mutable:milvus.proto.indexcgo.BinarySet.datas)
return datas_.Mutable(index);
}
inline ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::indexcgo::Binary >*
BinarySet::mutable_datas() {
// @@protoc_insertion_point(field_mutable_list:milvus.proto.indexcgo.BinarySet.datas)
return &datas_;
}
inline const ::milvus::proto::indexcgo::Binary& BinarySet::datas(int index) const {
// @@protoc_insertion_point(field_get:milvus.proto.indexcgo.BinarySet.datas)
return datas_.Get(index);
}
inline ::milvus::proto::indexcgo::Binary* BinarySet::add_datas() {
// @@protoc_insertion_point(field_add:milvus.proto.indexcgo.BinarySet.datas)
return datas_.Add();
}
inline const ::PROTOBUF_NAMESPACE_ID::RepeatedPtrField< ::milvus::proto::indexcgo::Binary >&
BinarySet::datas() const {
// @@protoc_insertion_point(field_list:milvus.proto.indexcgo.BinarySet.datas)
return datas_;
}
#ifdef __GNUC__
#pragma GCC diagnostic pop
#endif // __GNUC__
// -------------------------------------------------------------------
// -------------------------------------------------------------------
// -------------------------------------------------------------------
// @@protoc_insertion_point(namespace_scope)
} // namespace indexcgo
} // namespace proto
} // namespace milvus
// @@protoc_insertion_point(global_scope)
#include <google/protobuf/port_undef.inc>
#endif // GOOGLE_PROTOBUF_INCLUDED_GOOGLE_PROTOBUF_INCLUDED_index_5fcgo_5fmsg_2eproto

81
internal/indexbuilder/index.go
View File

@ -6,15 +6,18 @@ package indexbuilder
#cgo LDFLAGS: -L${SRCDIR}/../core/output/lib -lmilvus_indexbuilder -Wl,-rpath=${SRCDIR}/../core/output/lib
#include <stdlib.h> // free
#include "segcore/collection_c.h"
#include "indexbuilder/index_c.h"
*/
import "C"
import (
"encoding/json"
"unsafe"
"github.com/zilliztech/milvus-distributed/internal/errors"
"github.com/golang/protobuf/proto"
"github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
"github.com/zilliztech/milvus-distributed/internal/proto/indexcgopb"
)
// TODO: use storage.Blob instead later
@ -36,54 +39,94 @@ type CIndex struct {
}
func (index *CIndex) Serialize() ([]*Blob, error) {
var cDumpedSlicedBuffer *C.char = C.SerializeToSlicedBuffer(index.indexPtr)
var dumpedSlicedBuffer string = C.GoString(cDumpedSlicedBuffer)
/*
char*
SerializeToSlicedBuffer(CIndex index, int32_t* buffer_size);
*/
var bufferSize int32
var cDumpedSlicedBuffer *C.char = C.SerializeToSlicedBuffer(index.indexPtr, (*C.int32_t)(unsafe.Pointer(&bufferSize)))
defer C.free(unsafe.Pointer(cDumpedSlicedBuffer))
var data map[string]interface{}
err := json.Unmarshal([]byte(dumpedSlicedBuffer), &data)
dumpedSlicedBuffer := C.GoBytes(unsafe.Pointer(cDumpedSlicedBuffer), (C.int32_t)(bufferSize))
var blobs indexcgopb.BinarySet
err := proto.Unmarshal(dumpedSlicedBuffer, &blobs)
if err != nil {
return nil, errors.New("unmarshal sliced buffer failed")
return nil, err
}
ret := make([]*Blob, 0)
for key, value := range data {
valueString, ok := value.(string)
if !ok {
return nil, errors.New("unexpected data type of dumped sliced buffer")
}
ret = append(ret, &Blob{key, []byte(valueString)})
for _, data := range blobs.Datas {
ret = append(ret, &Blob{Key: data.Key, Value: data.Value})
}
return ret, nil
}
func (index *CIndex) Load([]*Blob) error {
func (index *CIndex) Load(blobs []*Blob) error {
binarySet := &indexcgopb.BinarySet{Datas: make([]*indexcgopb.Binary, 0)}
for _, blob := range blobs {
binarySet.Datas = append(binarySet.Datas, &indexcgopb.Binary{Key: blob.Key, Value: blob.Value})
}
datas, err := proto.Marshal(binarySet)
if err != nil {
return err
}
/*
void
LoadFromSlicedBuffer(CIndex index, const char* serialized_sliced_blob_buffer);
*/
C.LoadFromSlicedBuffer(index.indexPtr, (*C.char)(unsafe.Pointer(&datas[0])))
return nil
}
func (index *CIndex) BuildFloatVecIndex(vectors []float32) error {
/*
void
BuildFloatVecIndex(CIndex index, int64_t float_value_num, const float* vectors);
*/
C.BuildFloatVecIndex(index.indexPtr, (C.int64_t)(len(vectors)), (*C.float)(&vectors[0]))
return nil
}
func (index *CIndex) Delete() error {
/*
void
DeleteIndex(CIndex index);
*/
C.DeleteIndex(index.indexPtr)
return nil
}
func NewCIndex(typeParams, indexParams map[string]string) (Index, error) {
dumpedTypeParamsStr, err := json.Marshal(typeParams)
protoTypeParams := &indexcgopb.TypeParams{
Params: make([]*commonpb.KeyValuePair, 0),
}
for key, value := range typeParams {
protoTypeParams.Params = append(protoTypeParams.Params, &commonpb.KeyValuePair{Key: key, Value: value})
}
typeParamsStr, err := proto.Marshal(protoTypeParams)
if err != nil {
return nil, err
}
dumpedIndexParamsStr, err := json.Marshal(indexParams)
protoIndexParams := &indexcgopb.IndexParams{
Params: make([]*commonpb.KeyValuePair, 0),
}
for key, value := range indexParams {
protoIndexParams.Params = append(protoIndexParams.Params, &commonpb.KeyValuePair{Key: key, Value: value})
}
indexParamsStr, err := proto.Marshal(protoIndexParams)
if err != nil {
return nil, err
}
cDumpedTypeParamsStr := C.CString(string(dumpedTypeParamsStr))
cDumpedIndexParamsStr := C.CString(string(dumpedIndexParamsStr))
/*
CIndex
CreateIndex(const char* serialized_type_params, const char* serialized_index_params);
*/
return &CIndex{
indexPtr: C.CreateIndex(cDumpedTypeParamsStr, cDumpedIndexParamsStr),
indexPtr: C.CreateIndex((*C.char)(unsafe.Pointer(&typeParamsStr[0])), (*C.char)(unsafe.Pointer(&indexParamsStr[0]))),
}, nil
}

23
internal/proto/index_cgo_msg.proto Normal file
View File

@ -0,0 +1,23 @@
syntax = "proto3";
package milvus.proto.indexcgo;
option go_package="github.com/zilliztech/milvus-distributed/internal/proto/indexcgopb";
import "common.proto";
message TypeParams {
repeated common.KeyValuePair params = 1;
}
message IndexParams {
repeated common.KeyValuePair params = 1;
}
message Binary {
string key = 1;
bytes value = 2;
}
message BinarySet {
repeated Binary datas = 1;
}

216
internal/proto/indexcgopb/index_cgo_msg.pb.go Normal file
View File

@ -0,0 +1,216 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// source: index_cgo_msg.proto
package indexcgopb
import (
fmt "fmt"
proto "github.com/golang/protobuf/proto"
commonpb "github.com/zilliztech/milvus-distributed/internal/proto/commonpb"
math "math"
)
// Reference imports to suppress errors if they are not otherwise used.
var _ = proto.Marshal
var _ = fmt.Errorf
var _ = math.Inf
// This is a compile-time assertion to ensure that this generated file
// is compatible with the proto package it is being compiled against.
// A compilation error at this line likely means your copy of the
// proto package needs to be updated.
const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
type TypeParams struct {
Params []*commonpb.KeyValuePair `protobuf:"bytes,1,rep,name=params,proto3" json:"params,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *TypeParams) Reset() { *m = TypeParams{} }
func (m *TypeParams) String() string { return proto.CompactTextString(m) }
func (*TypeParams) ProtoMessage() {}
func (*TypeParams) Descriptor() ([]byte, []int) {
return fileDescriptor_c009bd9544a7343c, []int{0}
}
func (m *TypeParams) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_TypeParams.Unmarshal(m, b)
}
func (m *TypeParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_TypeParams.Marshal(b, m, deterministic)
}
func (m *TypeParams) XXX_Merge(src proto.Message) {
xxx_messageInfo_TypeParams.Merge(m, src)
}
func (m *TypeParams) XXX_Size() int {
return xxx_messageInfo_TypeParams.Size(m)
}
func (m *TypeParams) XXX_DiscardUnknown() {
xxx_messageInfo_TypeParams.DiscardUnknown(m)
}
var xxx_messageInfo_TypeParams proto.InternalMessageInfo
func (m *TypeParams) GetParams() []*commonpb.KeyValuePair {
if m != nil {
return m.Params
}
return nil
}
type IndexParams struct {
Params []*commonpb.KeyValuePair `protobuf:"bytes,1,rep,name=params,proto3" json:"params,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *IndexParams) Reset() { *m = IndexParams{} }
func (m *IndexParams) String() string { return proto.CompactTextString(m) }
func (*IndexParams) ProtoMessage() {}
func (*IndexParams) Descriptor() ([]byte, []int) {
return fileDescriptor_c009bd9544a7343c, []int{1}
}
func (m *IndexParams) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_IndexParams.Unmarshal(m, b)
}
func (m *IndexParams) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_IndexParams.Marshal(b, m, deterministic)
}
func (m *IndexParams) XXX_Merge(src proto.Message) {
xxx_messageInfo_IndexParams.Merge(m, src)
}
func (m *IndexParams) XXX_Size() int {
return xxx_messageInfo_IndexParams.Size(m)
}
func (m *IndexParams) XXX_DiscardUnknown() {
xxx_messageInfo_IndexParams.DiscardUnknown(m)
}
var xxx_messageInfo_IndexParams proto.InternalMessageInfo
func (m *IndexParams) GetParams() []*commonpb.KeyValuePair {
if m != nil {
return m.Params
}
return nil
}
type Binary struct {
Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *Binary) Reset() { *m = Binary{} }
func (m *Binary) String() string { return proto.CompactTextString(m) }
func (*Binary) ProtoMessage() {}
func (*Binary) Descriptor() ([]byte, []int) {
return fileDescriptor_c009bd9544a7343c, []int{2}
}
func (m *Binary) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_Binary.Unmarshal(m, b)
}
func (m *Binary) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_Binary.Marshal(b, m, deterministic)
}
func (m *Binary) XXX_Merge(src proto.Message) {
xxx_messageInfo_Binary.Merge(m, src)
}
func (m *Binary) XXX_Size() int {
return xxx_messageInfo_Binary.Size(m)
}
func (m *Binary) XXX_DiscardUnknown() {
xxx_messageInfo_Binary.DiscardUnknown(m)
}
var xxx_messageInfo_Binary proto.InternalMessageInfo
func (m *Binary) GetKey() string {
if m != nil {
return m.Key
}
return ""
}
func (m *Binary) GetValue() []byte {
if m != nil {
return m.Value
}
return nil
}
type BinarySet struct {
Datas []*Binary `protobuf:"bytes,1,rep,name=datas,proto3" json:"datas,omitempty"`
XXX_NoUnkeyedLiteral struct{} `json:"-"`
XXX_unrecognized []byte `json:"-"`
XXX_sizecache int32 `json:"-"`
}
func (m *BinarySet) Reset() { *m = BinarySet{} }
func (m *BinarySet) String() string { return proto.CompactTextString(m) }
func (*BinarySet) ProtoMessage() {}
func (*BinarySet) Descriptor() ([]byte, []int) {
return fileDescriptor_c009bd9544a7343c, []int{3}
}
func (m *BinarySet) XXX_Unmarshal(b []byte) error {
return xxx_messageInfo_BinarySet.Unmarshal(m, b)
}
func (m *BinarySet) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
return xxx_messageInfo_BinarySet.Marshal(b, m, deterministic)
}
func (m *BinarySet) XXX_Merge(src proto.Message) {
xxx_messageInfo_BinarySet.Merge(m, src)
}
func (m *BinarySet) XXX_Size() int {
return xxx_messageInfo_BinarySet.Size(m)
}
func (m *BinarySet) XXX_DiscardUnknown() {
xxx_messageInfo_BinarySet.DiscardUnknown(m)
}
var xxx_messageInfo_BinarySet proto.InternalMessageInfo
func (m *BinarySet) GetDatas() []*Binary {
if m != nil {
return m.Datas
}
return nil
}
func init() {
proto.RegisterType((*TypeParams)(nil), "milvus.proto.indexcgo.TypeParams")
proto.RegisterType((*IndexParams)(nil), "milvus.proto.indexcgo.IndexParams")
proto.RegisterType((*Binary)(nil), "milvus.proto.indexcgo.Binary")
proto.RegisterType((*BinarySet)(nil), "milvus.proto.indexcgo.BinarySet")
}
func init() { proto.RegisterFile("index_cgo_msg.proto", fileDescriptor_c009bd9544a7343c) }
var fileDescriptor_c009bd9544a7343c = []byte{
// 257 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x90, 0xbf, 0x4b, 0xc3, 0x40,
0x14, 0xc7, 0x89, 0xa5, 0x81, 0xbe, 0x76, 0x90, 0x53, 0x21, 0x08, 0x42, 0xcc, 0x94, 0xc5, 0x3b,
0xb1, 0x93, 0x9b, 0x04, 0x41, 0xc5, 0xa5, 0x44, 0x71, 0x70, 0x29, 0x97, 0xe4, 0x48, 0x1f, 0xde,
0x8f, 0x70, 0xb9, 0x14, 0xd3, 0xbf, 0x5e, 0x92, 0x6b, 0x06, 0xc1, 0xcd, 0xed, 0xbd, 0xc7, 0xf7,
0xf3, 0xf9, 0x1e, 0x07, 0x67, 0xa8, 0x2b, 0xf1, 0xbd, 0x2d, 0x6b, 0xb3, 0x55, 0x6d, 0x4d, 0x1b,
0x6b, 0x9c, 0x21, 0x17, 0x0a, 0xe5, 0xbe, 0x6b, 0xfd, 0x46, 0xc7, 0x44, 0x59, 0x9b, 0xcb, 0x55,
0x69, 0x94, 0x32, 0xda, 0x9f, 0x93, 0x27, 0x80, 0xf7, 0xbe, 0x11, 0x1b, 0x6e, 0xb9, 0x6a, 0xc9,
0x3d, 0x84, 0xcd, 0x38, 0x45, 0x41, 0x3c, 0x4b, 0x97, 0x77, 0xd7, 0xf4, 0x97, 0xe3, 0x48, 0xbe,
0x8a, 0xfe, 0x83, 0xcb, 0x4e, 0x6c, 0x38, 0xda, 0xfc, 0x08, 0x24, 0xcf, 0xb0, 0x7c, 0x19, 0x2a,
0xfe, 0x6f, 0xba, 0x85, 0x30, 0x43, 0xcd, 0x6d, 0x4f, 0x4e, 0x61, 0xf6, 0x25, 0xfa, 0x28, 0x88,
0x83, 0x74, 0x91, 0x0f, 0x23, 0x39, 0x87, 0xf9, 0x7e, 0x00, 0xa2, 0x93, 0x38, 0x48, 0x57, 0xb9,
0x5f, 0x92, 0x07, 0x58, 0x78, 0xe2, 0x4d, 0x38, 0xb2, 0x86, 0x79, 0xc5, 0x1d, 0x9f, 0x8a, 0xaf,
0xe8, 0x9f, 0xdf, 0x40, 0x3d, 0x90, 0xfb, 0x6c, 0xf6, 0xf8, 0x99, 0xd5, 0xe8, 0x76, 0x5d, 0x31,
0xbc, 0x8c, 0x1d, 0x50, 0x4a, 0x3c, 0x38, 0x51, 0xee, 0x98, 0x87, 0x6f, 0x2a, 0x6c, 0x9d, 0xc5,
0xa2, 0x73, 0xa2, 0x62, 0xa8, 0x9d, 0xb0, 0x9a, 0x4b, 0x36, 0x1a, 0xd9, 0x64, 0x6c, 0x8a, 0x22,
0x1c, 0x2f, 0xeb, 0x9f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xb4, 0xb0, 0x70, 0x01, 0x8f, 0x01, 0x00,
0x00,
}

2
scripts/proto_gen_go.sh
View File

@ -24,6 +24,7 @@ mkdir -p internalpb
mkdir -p servicepb
mkdir -p masterpb
mkdir -p indexbuilderpb
mkdir -p indexcgopb
mkdir -p writerpb
${protoc} --go_out=plugins=grpc,paths=source_relative:./commonpb common.proto
@ -34,6 +35,7 @@ ${protoc} --go_out=plugins=grpc,paths=source_relative:./servicepb service_msg.pr
${protoc} --go_out=plugins=grpc,paths=source_relative:./servicepb service.proto
${protoc} --go_out=plugins=grpc,paths=source_relative:./masterpb master.proto
${protoc} --go_out=plugins=grpc,paths=source_relative:./indexbuilderpb index_builder.proto
${protoc} --go_out=plugins=grpc,paths=source_relative:./indexcgopb index_cgo_msg.proto
${protoc} --go_out=plugins=grpc,paths=source_relative:./writerpb write_node.proto
popd