Add float16 vector (#25852)

Signed-off-by: Writer-X <1256866856@qq.com>
This commit is contained in:
Xu Tong 2023-09-08 10:03:16 +08:00 committed by GitHub
parent eb3ea3a1df
commit 9166011c4a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
87 changed files with 2154 additions and 401 deletions

View File

@ -49,6 +49,9 @@ datatype_sizeof(DataType data_type, int dim = 1) {
AssertInfo(dim % 8 == 0, "dim=" + std::to_string(dim)); AssertInfo(dim % 8 == 0, "dim=" + std::to_string(dim));
return dim / 8; return dim / 8;
} }
case DataType::VECTOR_FLOAT16: {
return sizeof(float16) * dim;
}
default: { default: {
throw std::invalid_argument("unsupported data type"); throw std::invalid_argument("unsupported data type");
} }
@ -84,6 +87,9 @@ datatype_name(DataType data_type) {
case DataType::VECTOR_BINARY: { case DataType::VECTOR_BINARY: {
return "vector_binary"; return "vector_binary";
} }
case DataType::VECTOR_FLOAT16: {
return "vector_float16";
}
default: { default: {
auto err_msg = auto err_msg =
"Unsupported DataType(" + std::to_string((int)data_type) + ")"; "Unsupported DataType(" + std::to_string((int)data_type) + ")";
@ -95,7 +101,8 @@ datatype_name(DataType data_type) {
inline bool inline bool
datatype_is_vector(DataType datatype) { datatype_is_vector(DataType datatype) {
return datatype == DataType::VECTOR_BINARY || return datatype == DataType::VECTOR_BINARY ||
datatype == DataType::VECTOR_FLOAT; datatype == DataType::VECTOR_FLOAT ||
datatype == DataType::VECTOR_FLOAT16;
} }
inline bool inline bool

View File

@ -51,6 +51,33 @@ using offset_t = int32_t;
using date_t = int32_t; using date_t = int32_t;
using distance_t = float; using distance_t = float;
union float16 {
unsigned short bits;
struct {
unsigned short mantissa : 10;
unsigned short exponent : 5;
unsigned short sign : 1;
} parts;
float16() {
}
float16(float f) {
unsigned int i = *(unsigned int*)&f;
unsigned int sign = (i >> 31) & 0x0001;
unsigned int exponent = ((i >> 23) & 0xff) - 127 + 15;
unsigned int mantissa = (i >> 13) & 0x3ff;
parts.sign = sign;
parts.exponent = exponent;
parts.mantissa = mantissa;
}
operator float() const {
unsigned int sign = parts.sign << 31;
unsigned int exponent = (parts.exponent - 15 + 127) << 23;
unsigned int mantissa = parts.mantissa << 13;
unsigned int bits = sign | exponent | mantissa;
return *(float*)&bits;
}
};
enum class DataType { enum class DataType {
NONE = 0, NONE = 0,
BOOL = 1, BOOL = 1,
@ -69,6 +96,7 @@ enum class DataType {
VECTOR_BINARY = 100, VECTOR_BINARY = 100,
VECTOR_FLOAT = 101, VECTOR_FLOAT = 101,
VECTOR_FLOAT16 = 102,
}; };
using Timestamp = uint64_t; // TODO: use TiKV-like timestamp using Timestamp = uint64_t; // TODO: use TiKV-like timestamp
@ -202,6 +230,9 @@ struct fmt::formatter<milvus::DataType> : formatter<string_view> {
case milvus::DataType::VECTOR_FLOAT: case milvus::DataType::VECTOR_FLOAT:
name = "VECTOR_FLOAT"; name = "VECTOR_FLOAT";
break; break;
case milvus::DataType::VECTOR_FLOAT16:
name = "VECTOR_FLOAT16";
break;
} }
return formatter<string_view>::format(name, ctx); return formatter<string_view>::format(name, ctx);
} }

View File

@ -35,12 +35,20 @@ class BinaryVector : public VectorTrait {
static constexpr auto metric_type = DataType::VECTOR_BINARY; static constexpr auto metric_type = DataType::VECTOR_BINARY;
}; };
class Float16Vector : public VectorTrait {
public:
using embedded_type = float16;
static constexpr auto metric_type = DataType::VECTOR_FLOAT16;
};
template <typename VectorType> template <typename VectorType>
inline constexpr int64_t inline constexpr int64_t
element_sizeof(int64_t dim) { element_sizeof(int64_t dim) {
static_assert(std::is_base_of_v<VectorType, VectorTrait>); static_assert(std::is_base_of_v<VectorType, VectorTrait>);
if constexpr (std::is_same_v<VectorType, FloatVector>) { if constexpr (std::is_same_v<VectorType, FloatVector>) {
return dim * sizeof(float); return dim * sizeof(float);
} else if constexpr (std::is_same_v<VectorType, Float16Vector>) {
return dim * sizeof(float16);
} else { } else {
return dim / 8; return dim / 8;
} }
@ -64,8 +72,10 @@ struct EmbeddedTypeImpl<T, std::enable_if_t<IsScalar<T>>> {
template <typename T> template <typename T>
struct EmbeddedTypeImpl<T, std::enable_if_t<IsVector<T>>> { struct EmbeddedTypeImpl<T, std::enable_if_t<IsVector<T>>> {
using type = using type = std::conditional_t<
std::conditional_t<std::is_same_v<T, FloatVector>, float, uint8_t>; std::is_same_v<T, FloatVector>,
float,
std::conditional_t<std::is_same_v<T, Float16Vector>, float16, uint8_t>>;
}; };
template <typename T> template <typename T>

View File

@ -56,6 +56,7 @@ enum CDataType {
BinaryVector = 100, BinaryVector = 100,
FloatVector = 101, FloatVector = 101,
Float16Vector = 102,
}; };
typedef enum CDataType CDataType; typedef enum CDataType CDataType;

View File

@ -309,7 +309,7 @@ PROTOBUF_CONSTEXPR VectorANNS::VectorANNS(
, /*decltype(_impl_.predicates_)*/nullptr , /*decltype(_impl_.predicates_)*/nullptr
, /*decltype(_impl_.query_info_)*/nullptr , /*decltype(_impl_.query_info_)*/nullptr
, /*decltype(_impl_.field_id_)*/int64_t{0} , /*decltype(_impl_.field_id_)*/int64_t{0}
, /*decltype(_impl_.is_binary_)*/false , /*decltype(_impl_.vector_type_)*/0
, /*decltype(_impl_._cached_size_)*/{}} {} , /*decltype(_impl_._cached_size_)*/{}} {}
struct VectorANNSDefaultTypeInternal { struct VectorANNSDefaultTypeInternal {
PROTOBUF_CONSTEXPR VectorANNSDefaultTypeInternal() PROTOBUF_CONSTEXPR VectorANNSDefaultTypeInternal()
@ -355,7 +355,7 @@ PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORIT
} // namespace proto } // namespace proto
} // namespace milvus } // namespace milvus
static ::_pb::Metadata file_level_metadata_plan_2eproto[22]; static ::_pb::Metadata file_level_metadata_plan_2eproto[22];
static const ::_pb::EnumDescriptor* file_level_enum_descriptors_plan_2eproto[5]; static const ::_pb::EnumDescriptor* file_level_enum_descriptors_plan_2eproto[6];
static constexpr ::_pb::ServiceDescriptor const** file_level_service_descriptors_plan_2eproto = nullptr; static constexpr ::_pb::ServiceDescriptor const** file_level_service_descriptors_plan_2eproto = nullptr;
const uint32_t TableStruct_plan_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = { const uint32_t TableStruct_plan_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(protodesc_cold) = {
@ -548,7 +548,7 @@ const uint32_t TableStruct_plan_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(pro
~0u, // no _oneof_case_ ~0u, // no _oneof_case_
~0u, // no _weak_field_map_ ~0u, // no _weak_field_map_
~0u, // no _inlined_string_donated_ ~0u, // no _inlined_string_donated_
PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::VectorANNS, _impl_.is_binary_), PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::VectorANNS, _impl_.vector_type_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::VectorANNS, _impl_.field_id_), PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::VectorANNS, _impl_.field_id_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::VectorANNS, _impl_.predicates_), PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::VectorANNS, _impl_.predicates_),
PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::VectorANNS, _impl_.query_info_), PROTOBUF_FIELD_OFFSET(::milvus::proto::plan::VectorANNS, _impl_.query_info_),
@ -711,33 +711,36 @@ const char descriptor_table_protodef_plan_2eproto[] PROTOBUF_SECTION_VARIABLE(pr
"_true_expr\030\014 \001(\0132!.milvus.proto.plan.Alw" "_true_expr\030\014 \001(\0132!.milvus.proto.plan.Alw"
"aysTrueExprH\000\022A\n\022json_contains_expr\030\r \001(" "aysTrueExprH\000\022A\n\022json_contains_expr\030\r \001("
"\0132#.milvus.proto.plan.JSONContainsExprH\000" "\0132#.milvus.proto.plan.JSONContainsExprH\000"
"B\006\n\004expr\"\251\001\n\nVectorANNS\022\021\n\tis_binary\030\001 \001" "B\006\n\004expr\"\312\001\n\nVectorANNS\0222\n\013vector_type\030\001"
"(\010\022\020\n\010field_id\030\002 \001(\003\022+\n\npredicates\030\003 \001(\013" " \001(\0162\035.milvus.proto.plan.VectorType\022\020\n\010f"
"2\027.milvus.proto.plan.Expr\0220\n\nquery_info\030" "ield_id\030\002 \001(\003\022+\n\npredicates\030\003 \001(\0132\027.milv"
"\004 \001(\0132\034.milvus.proto.plan.QueryInfo\022\027\n\017p" "us.proto.plan.Expr\0220\n\nquery_info\030\004 \001(\0132\034"
"laceholder_tag\030\005 \001(\t\"]\n\rQueryPlanNode\022+\n" ".milvus.proto.plan.QueryInfo\022\027\n\017placehol"
"\npredicates\030\001 \001(\0132\027.milvus.proto.plan.Ex" "der_tag\030\005 \001(\t\"]\n\rQueryPlanNode\022+\n\npredic"
"pr\022\020\n\010is_count\030\002 \001(\010\022\r\n\005limit\030\003 \001(\003\"\304\001\n\010" "ates\030\001 \001(\0132\027.milvus.proto.plan.Expr\022\020\n\010i"
"PlanNode\0224\n\013vector_anns\030\001 \001(\0132\035.milvus.p" "s_count\030\002 \001(\010\022\r\n\005limit\030\003 \001(\003\"\304\001\n\010PlanNod"
"roto.plan.VectorANNSH\000\022-\n\npredicates\030\002 \001" "e\0224\n\013vector_anns\030\001 \001(\0132\035.milvus.proto.pl"
"(\0132\027.milvus.proto.plan.ExprH\000\0221\n\005query\030\004" "an.VectorANNSH\000\022-\n\npredicates\030\002 \001(\0132\027.mi"
" \001(\0132 .milvus.proto.plan.QueryPlanNodeH\000" "lvus.proto.plan.ExprH\000\0221\n\005query\030\004 \001(\0132 ."
"\022\030\n\020output_field_ids\030\003 \003(\003B\006\n\004node*\272\001\n\006O" "milvus.proto.plan.QueryPlanNodeH\000\022\030\n\020out"
"pType\022\013\n\007Invalid\020\000\022\017\n\013GreaterThan\020\001\022\020\n\014G" "put_field_ids\030\003 \003(\003B\006\n\004node*\272\001\n\006OpType\022\013"
"reaterEqual\020\002\022\014\n\010LessThan\020\003\022\r\n\tLessEqual" "\n\007Invalid\020\000\022\017\n\013GreaterThan\020\001\022\020\n\014GreaterE"
"\020\004\022\t\n\005Equal\020\005\022\014\n\010NotEqual\020\006\022\017\n\013PrefixMat" "qual\020\002\022\014\n\010LessThan\020\003\022\r\n\tLessEqual\020\004\022\t\n\005E"
"ch\020\007\022\020\n\014PostfixMatch\020\010\022\t\n\005Match\020\t\022\t\n\005Ran" "qual\020\005\022\014\n\010NotEqual\020\006\022\017\n\013PrefixMatch\020\007\022\020\n"
"ge\020\n\022\006\n\002In\020\013\022\t\n\005NotIn\020\014*G\n\013ArithOpType\022\013" "\014PostfixMatch\020\010\022\t\n\005Match\020\t\022\t\n\005Range\020\n\022\006\n"
"\n\007Unknown\020\000\022\007\n\003Add\020\001\022\007\n\003Sub\020\002\022\007\n\003Mul\020\003\022\007" "\002In\020\013\022\t\n\005NotIn\020\014*G\n\013ArithOpType\022\013\n\007Unkno"
"\n\003Div\020\004\022\007\n\003Mod\020\005B3Z1github.com/milvus-io" "wn\020\000\022\007\n\003Add\020\001\022\007\n\003Sub\020\002\022\007\n\003Mul\020\003\022\007\n\003Div\020\004"
"/milvus/internal/proto/planpbb\006proto3" "\022\007\n\003Mod\020\005*B\n\nVectorType\022\020\n\014BinaryVector\020"
"\000\022\017\n\013FloatVector\020\001\022\021\n\rFloat16Vector\020\002B3Z"
"1github.com/milvus-io/milvus/internal/pr"
"oto/planpbb\006proto3"
; ;
static const ::_pbi::DescriptorTable* const descriptor_table_plan_2eproto_deps[1] = { static const ::_pbi::DescriptorTable* const descriptor_table_plan_2eproto_deps[1] = {
&::descriptor_table_schema_2eproto, &::descriptor_table_schema_2eproto,
}; };
static ::_pbi::once_flag descriptor_table_plan_2eproto_once; static ::_pbi::once_flag descriptor_table_plan_2eproto_once;
const ::_pbi::DescriptorTable descriptor_table_plan_2eproto = { const ::_pbi::DescriptorTable descriptor_table_plan_2eproto = {
false, false, 4237, descriptor_table_protodef_plan_2eproto, false, false, 4338, descriptor_table_protodef_plan_2eproto,
"plan.proto", "plan.proto",
&descriptor_table_plan_2eproto_once, descriptor_table_plan_2eproto_deps, 1, 22, &descriptor_table_plan_2eproto_once, descriptor_table_plan_2eproto_deps, 1, 22,
schemas, file_default_instances, TableStruct_plan_2eproto::offsets, schemas, file_default_instances, TableStruct_plan_2eproto::offsets,
@ -865,6 +868,21 @@ bool ArithOpType_IsValid(int value) {
} }
} }
const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* VectorType_descriptor() {
::PROTOBUF_NAMESPACE_ID::internal::AssignDescriptors(&descriptor_table_plan_2eproto);
return file_level_enum_descriptors_plan_2eproto[5];
}
bool VectorType_IsValid(int value) {
switch (value) {
case 0:
case 1:
case 2:
return true;
default:
return false;
}
}
// =================================================================== // ===================================================================
@ -6475,7 +6493,7 @@ VectorANNS::VectorANNS(const VectorANNS& from)
, decltype(_impl_.predicates_){nullptr} , decltype(_impl_.predicates_){nullptr}
, decltype(_impl_.query_info_){nullptr} , decltype(_impl_.query_info_){nullptr}
, decltype(_impl_.field_id_){} , decltype(_impl_.field_id_){}
, decltype(_impl_.is_binary_){} , decltype(_impl_.vector_type_){}
, /*decltype(_impl_._cached_size_)*/{}}; , /*decltype(_impl_._cached_size_)*/{}};
_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_);
@ -6494,8 +6512,8 @@ VectorANNS::VectorANNS(const VectorANNS& from)
_this->_impl_.query_info_ = new ::milvus::proto::plan::QueryInfo(*from._impl_.query_info_); _this->_impl_.query_info_ = new ::milvus::proto::plan::QueryInfo(*from._impl_.query_info_);
} }
::memcpy(&_impl_.field_id_, &from._impl_.field_id_, ::memcpy(&_impl_.field_id_, &from._impl_.field_id_,
static_cast<size_t>(reinterpret_cast<char*>(&_impl_.is_binary_) - static_cast<size_t>(reinterpret_cast<char*>(&_impl_.vector_type_) -
reinterpret_cast<char*>(&_impl_.field_id_)) + sizeof(_impl_.is_binary_)); reinterpret_cast<char*>(&_impl_.field_id_)) + sizeof(_impl_.vector_type_));
// @@protoc_insertion_point(copy_constructor:milvus.proto.plan.VectorANNS) // @@protoc_insertion_point(copy_constructor:milvus.proto.plan.VectorANNS)
} }
@ -6508,7 +6526,7 @@ inline void VectorANNS::SharedCtor(
, decltype(_impl_.predicates_){nullptr} , decltype(_impl_.predicates_){nullptr}
, decltype(_impl_.query_info_){nullptr} , decltype(_impl_.query_info_){nullptr}
, decltype(_impl_.field_id_){int64_t{0}} , decltype(_impl_.field_id_){int64_t{0}}
, decltype(_impl_.is_binary_){false} , decltype(_impl_.vector_type_){0}
, /*decltype(_impl_._cached_size_)*/{} , /*decltype(_impl_._cached_size_)*/{}
}; };
_impl_.placeholder_tag_.InitDefault(); _impl_.placeholder_tag_.InitDefault();
@ -6553,8 +6571,8 @@ void VectorANNS::Clear() {
} }
_impl_.query_info_ = nullptr; _impl_.query_info_ = nullptr;
::memset(&_impl_.field_id_, 0, static_cast<size_t>( ::memset(&_impl_.field_id_, 0, static_cast<size_t>(
reinterpret_cast<char*>(&_impl_.is_binary_) - reinterpret_cast<char*>(&_impl_.vector_type_) -
reinterpret_cast<char*>(&_impl_.field_id_)) + sizeof(_impl_.is_binary_)); reinterpret_cast<char*>(&_impl_.field_id_)) + sizeof(_impl_.vector_type_));
_internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>();
} }
@ -6564,11 +6582,12 @@ const char* VectorANNS::_InternalParse(const char* ptr, ::_pbi::ParseContext* ct
uint32_t tag; uint32_t tag;
ptr = ::_pbi::ReadTag(ptr, &tag); ptr = ::_pbi::ReadTag(ptr, &tag);
switch (tag >> 3) { switch (tag >> 3) {
// bool is_binary = 1; // .milvus.proto.plan.VectorType vector_type = 1;
case 1: case 1:
if (PROTOBUF_PREDICT_TRUE(static_cast<uint8_t>(tag) == 8)) { if (PROTOBUF_PREDICT_TRUE(static_cast<uint8_t>(tag) == 8)) {
_impl_.is_binary_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); uint64_t val = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr);
CHK_(ptr); CHK_(ptr);
_internal_set_vector_type(static_cast<::milvus::proto::plan::VectorType>(val));
} else } else
goto handle_unusual; goto handle_unusual;
continue; continue;
@ -6635,10 +6654,11 @@ uint8_t* VectorANNS::_InternalSerialize(
uint32_t cached_has_bits = 0; uint32_t cached_has_bits = 0;
(void) cached_has_bits; (void) cached_has_bits;
// bool is_binary = 1; // .milvus.proto.plan.VectorType vector_type = 1;
if (this->_internal_is_binary() != 0) { if (this->_internal_vector_type() != 0) {
target = stream->EnsureSpace(target); target = stream->EnsureSpace(target);
target = ::_pbi::WireFormatLite::WriteBoolToArray(1, this->_internal_is_binary(), target); target = ::_pbi::WireFormatLite::WriteEnumToArray(
1, this->_internal_vector_type(), target);
} }
// int64 field_id = 2; // int64 field_id = 2;
@ -6713,9 +6733,10 @@ size_t VectorANNS::ByteSizeLong() const {
total_size += ::_pbi::WireFormatLite::Int64SizePlusOne(this->_internal_field_id()); total_size += ::_pbi::WireFormatLite::Int64SizePlusOne(this->_internal_field_id());
} }
// bool is_binary = 1; // .milvus.proto.plan.VectorType vector_type = 1;
if (this->_internal_is_binary() != 0) { if (this->_internal_vector_type() != 0) {
total_size += 1 + 1; total_size += 1 +
::_pbi::WireFormatLite::EnumSize(this->_internal_vector_type());
} }
return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_); return MaybeComputeUnknownFieldsSize(total_size, &_impl_._cached_size_);
@ -6750,8 +6771,8 @@ void VectorANNS::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PRO
if (from._internal_field_id() != 0) { if (from._internal_field_id() != 0) {
_this->_internal_set_field_id(from._internal_field_id()); _this->_internal_set_field_id(from._internal_field_id());
} }
if (from._internal_is_binary() != 0) { if (from._internal_vector_type() != 0) {
_this->_internal_set_is_binary(from._internal_is_binary()); _this->_internal_set_vector_type(from._internal_vector_type());
} }
_this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_); _this->_internal_metadata_.MergeFrom<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(from._internal_metadata_);
} }
@ -6777,8 +6798,8 @@ void VectorANNS::InternalSwap(VectorANNS* other) {
&other->_impl_.placeholder_tag_, rhs_arena &other->_impl_.placeholder_tag_, rhs_arena
); );
::PROTOBUF_NAMESPACE_ID::internal::memswap< ::PROTOBUF_NAMESPACE_ID::internal::memswap<
PROTOBUF_FIELD_OFFSET(VectorANNS, _impl_.is_binary_) PROTOBUF_FIELD_OFFSET(VectorANNS, _impl_.vector_type_)
+ sizeof(VectorANNS::_impl_.is_binary_) + sizeof(VectorANNS::_impl_.vector_type_)
- PROTOBUF_FIELD_OFFSET(VectorANNS, _impl_.predicates_)>( - PROTOBUF_FIELD_OFFSET(VectorANNS, _impl_.predicates_)>(
reinterpret_cast<char*>(&_impl_.predicates_), reinterpret_cast<char*>(&_impl_.predicates_),
reinterpret_cast<char*>(&other->_impl_.predicates_)); reinterpret_cast<char*>(&other->_impl_.predicates_));

View File

@ -290,6 +290,32 @@ inline bool ArithOpType_Parse(
return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum<ArithOpType>( return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum<ArithOpType>(
ArithOpType_descriptor(), name, value); ArithOpType_descriptor(), name, value);
} }
enum VectorType : int {
BinaryVector = 0,
FloatVector = 1,
Float16Vector = 2,
VectorType_INT_MIN_SENTINEL_DO_NOT_USE_ = std::numeric_limits<int32_t>::min(),
VectorType_INT_MAX_SENTINEL_DO_NOT_USE_ = std::numeric_limits<int32_t>::max()
};
bool VectorType_IsValid(int value);
constexpr VectorType VectorType_MIN = BinaryVector;
constexpr VectorType VectorType_MAX = Float16Vector;
constexpr int VectorType_ARRAYSIZE = VectorType_MAX + 1;
const ::PROTOBUF_NAMESPACE_ID::EnumDescriptor* VectorType_descriptor();
template<typename T>
inline const std::string& VectorType_Name(T enum_t_value) {
static_assert(::std::is_same<T, VectorType>::value ||
::std::is_integral<T>::value,
"Incorrect type passed to function VectorType_Name.");
return ::PROTOBUF_NAMESPACE_ID::internal::NameOfEnum(
VectorType_descriptor(), enum_t_value);
}
inline bool VectorType_Parse(
::PROTOBUF_NAMESPACE_ID::ConstStringParam name, VectorType* value) {
return ::PROTOBUF_NAMESPACE_ID::internal::ParseNamedEnum<VectorType>(
VectorType_descriptor(), name, value);
}
// =================================================================== // ===================================================================
class GenericValue final : class GenericValue final :
@ -4297,7 +4323,7 @@ class VectorANNS final :
kPredicatesFieldNumber = 3, kPredicatesFieldNumber = 3,
kQueryInfoFieldNumber = 4, kQueryInfoFieldNumber = 4,
kFieldIdFieldNumber = 2, kFieldIdFieldNumber = 2,
kIsBinaryFieldNumber = 1, kVectorTypeFieldNumber = 1,
}; };
// string placeholder_tag = 5; // string placeholder_tag = 5;
void clear_placeholder_tag(); void clear_placeholder_tag();
@ -4358,13 +4384,13 @@ class VectorANNS final :
void _internal_set_field_id(int64_t value); void _internal_set_field_id(int64_t value);
public: public:
// bool is_binary = 1; // .milvus.proto.plan.VectorType vector_type = 1;
void clear_is_binary(); void clear_vector_type();
bool is_binary() const; ::milvus::proto::plan::VectorType vector_type() const;
void set_is_binary(bool value); void set_vector_type(::milvus::proto::plan::VectorType value);
private: private:
bool _internal_is_binary() const; ::milvus::proto::plan::VectorType _internal_vector_type() const;
void _internal_set_is_binary(bool value); void _internal_set_vector_type(::milvus::proto::plan::VectorType value);
public: public:
// @@protoc_insertion_point(class_scope:milvus.proto.plan.VectorANNS) // @@protoc_insertion_point(class_scope:milvus.proto.plan.VectorANNS)
@ -4379,7 +4405,7 @@ class VectorANNS final :
::milvus::proto::plan::Expr* predicates_; ::milvus::proto::plan::Expr* predicates_;
::milvus::proto::plan::QueryInfo* query_info_; ::milvus::proto::plan::QueryInfo* query_info_;
int64_t field_id_; int64_t field_id_;
bool is_binary_; int vector_type_;
mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_; mutable ::PROTOBUF_NAMESPACE_ID::internal::CachedSize _cached_size_;
}; };
union { Impl_ _impl_; }; union { Impl_ _impl_; };
@ -8834,24 +8860,24 @@ inline Expr::ExprCase Expr::expr_case() const {
// VectorANNS // VectorANNS
// bool is_binary = 1; // .milvus.proto.plan.VectorType vector_type = 1;
inline void VectorANNS::clear_is_binary() { inline void VectorANNS::clear_vector_type() {
_impl_.is_binary_ = false; _impl_.vector_type_ = 0;
} }
inline bool VectorANNS::_internal_is_binary() const { inline ::milvus::proto::plan::VectorType VectorANNS::_internal_vector_type() const {
return _impl_.is_binary_; return static_cast< ::milvus::proto::plan::VectorType >(_impl_.vector_type_);
} }
inline bool VectorANNS::is_binary() const { inline ::milvus::proto::plan::VectorType VectorANNS::vector_type() const {
// @@protoc_insertion_point(field_get:milvus.proto.plan.VectorANNS.is_binary) // @@protoc_insertion_point(field_get:milvus.proto.plan.VectorANNS.vector_type)
return _internal_is_binary(); return _internal_vector_type();
} }
inline void VectorANNS::_internal_set_is_binary(bool value) { inline void VectorANNS::_internal_set_vector_type(::milvus::proto::plan::VectorType value) {
_impl_.is_binary_ = value; _impl_.vector_type_ = value;
} }
inline void VectorANNS::set_is_binary(bool value) { inline void VectorANNS::set_vector_type(::milvus::proto::plan::VectorType value) {
_internal_set_is_binary(value); _internal_set_vector_type(value);
// @@protoc_insertion_point(field_set:milvus.proto.plan.VectorANNS.is_binary) // @@protoc_insertion_point(field_set:milvus.proto.plan.VectorANNS.vector_type)
} }
// int64 field_id = 2; // int64 field_id = 2;
@ -9599,6 +9625,11 @@ template <>
inline const EnumDescriptor* GetEnumDescriptor< ::milvus::proto::plan::ArithOpType>() { inline const EnumDescriptor* GetEnumDescriptor< ::milvus::proto::plan::ArithOpType>() {
return ::milvus::proto::plan::ArithOpType_descriptor(); return ::milvus::proto::plan::ArithOpType_descriptor();
} }
template <> struct is_proto_enum< ::milvus::proto::plan::VectorType> : ::std::true_type {};
template <>
inline const EnumDescriptor* GetEnumDescriptor< ::milvus::proto::plan::VectorType>() {
return ::milvus::proto::plan::VectorType_descriptor();
}
PROTOBUF_NAMESPACE_CLOSE PROTOBUF_NAMESPACE_CLOSE

View File

@ -52,6 +52,12 @@ struct BinaryVectorANNS : VectorPlanNode {
accept(PlanNodeVisitor&) override; accept(PlanNodeVisitor&) override;
}; };
struct Float16VectorANNS : VectorPlanNode {
public:
void
accept(PlanNodeVisitor&) override;
};
struct RetrievePlanNode : PlanNode { struct RetrievePlanNode : PlanNode {
public: public:
void void

View File

@ -185,8 +185,12 @@ ProtoParser::PlanNodeFromProto(const planpb::PlanNode& plan_node_proto) {
search_info.search_params_ = json::parse(query_info_proto.search_params()); search_info.search_params_ = json::parse(query_info_proto.search_params());
auto plan_node = [&]() -> std::unique_ptr<VectorPlanNode> { auto plan_node = [&]() -> std::unique_ptr<VectorPlanNode> {
if (anns_proto.is_binary()) { if (anns_proto.vector_type() ==
milvus::proto::plan::VectorType::BinaryVector) {
return std::make_unique<BinaryVectorANNS>(); return std::make_unique<BinaryVectorANNS>();
} else if (anns_proto.vector_type() ==
milvus::proto::plan::VectorType::Float16Vector) {
return std::make_unique<Float16VectorANNS>();
} else { } else {
return std::make_unique<FloatVectorANNS>(); return std::make_unique<FloatVectorANNS>();
} }

View File

@ -30,7 +30,8 @@ CheckBruteForceSearchParam(const FieldMeta& field,
AssertInfo(datatype_is_vector(data_type), AssertInfo(datatype_is_vector(data_type),
"[BruteForceSearch] Data type isn't vector type"); "[BruteForceSearch] Data type isn't vector type");
bool is_float_data_type = (data_type == DataType::VECTOR_FLOAT); bool is_float_data_type = (data_type == DataType::VECTOR_FLOAT ||
data_type == DataType::VECTOR_FLOAT16);
bool is_float_metric_type = IsFloatMetricType(metric_type); bool is_float_metric_type = IsFloatMetricType(metric_type);
AssertInfo(is_float_data_type == is_float_metric_type, AssertInfo(is_float_data_type == is_float_metric_type,
"[BruteForceSearch] Data type and metric type miss-match"); "[BruteForceSearch] Data type and metric type miss-match");
@ -41,7 +42,8 @@ BruteForceSearch(const dataset::SearchDataset& dataset,
const void* chunk_data_raw, const void* chunk_data_raw,
int64_t chunk_rows, int64_t chunk_rows,
const knowhere::Json& conf, const knowhere::Json& conf,
const BitsetView& bitset) { const BitsetView& bitset,
DataType data_type) {
SubSearchResult sub_result(dataset.num_queries, SubSearchResult sub_result(dataset.num_queries,
dataset.topk, dataset.topk,
dataset.metric_type, dataset.metric_type,
@ -54,6 +56,38 @@ BruteForceSearch(const dataset::SearchDataset& dataset,
auto base_dataset = auto base_dataset =
knowhere::GenDataSet(chunk_rows, dim, chunk_data_raw); knowhere::GenDataSet(chunk_rows, dim, chunk_data_raw);
auto query_dataset = knowhere::GenDataSet(nq, dim, dataset.query_data); auto query_dataset = knowhere::GenDataSet(nq, dim, dataset.query_data);
if (data_type == DataType::VECTOR_FLOAT16) {
// Todo: Temporarily use cast to float32 to achieve, need to optimize
// first, First, transfer the cast to knowhere part
// second, knowhere partially supports float16 and removes the forced conversion to float32
auto xb = base_dataset->GetTensor();
std::vector<float> float_xb(base_dataset->GetRows() *
base_dataset->GetDim());
auto xq = query_dataset->GetTensor();
std::vector<float> float_xq(query_dataset->GetRows() *
query_dataset->GetDim());
auto fp16_xb = (const float16*)xb;
for (int i = 0;
i < base_dataset->GetRows() * base_dataset->GetDim();
i++) {
float_xb[i] = (float)fp16_xb[i];
}
auto fp16_xq = (const float16*)xq;
for (int i = 0;
i < query_dataset->GetRows() * query_dataset->GetDim();
i++) {
float_xq[i] = (float)fp16_xq[i];
}
void* void_ptr_xb = static_cast<void*>(float_xb.data());
void* void_ptr_xq = static_cast<void*>(float_xq.data());
base_dataset = knowhere::GenDataSet(chunk_rows, dim, void_ptr_xb);
query_dataset = knowhere::GenDataSet(nq, dim, void_ptr_xq);
}
auto config = knowhere::Json{ auto config = knowhere::Json{
{knowhere::meta::METRIC_TYPE, dataset.metric_type}, {knowhere::meta::METRIC_TYPE, dataset.metric_type},
{knowhere::meta::DIM, dim}, {knowhere::meta::DIM, dim},

View File

@ -28,6 +28,7 @@ BruteForceSearch(const dataset::SearchDataset& dataset,
const void* chunk_data_raw, const void* chunk_data_raw,
int64_t chunk_rows, int64_t chunk_rows,
const knowhere::Json& conf, const knowhere::Json& conf,
const BitsetView& bitset); const BitsetView& bitset,
DataType data_type = DataType::VECTOR_FLOAT);
} // namespace milvus::query } // namespace milvus::query

View File

@ -123,7 +123,9 @@ SearchOnGrowing(const segcore::SegmentGrowingImpl& segment,
chunk_data, chunk_data,
size_per_chunk, size_per_chunk,
info.search_params_, info.search_params_,
sub_view); sub_view,
data_type);
// convert chunk uid to segment uid // convert chunk uid to segment uid
for (auto& x : sub_qr.mutable_seg_offsets()) { for (auto& x : sub_qr.mutable_seg_offsets()) {
if (x != -1) { if (x != -1) {

View File

@ -84,9 +84,14 @@ SearchOnSealed(const Schema& schema,
field.get_dim(), field.get_dim(),
query_data}; query_data};
auto data_type = field.get_data_type();
CheckBruteForceSearchParam(field, search_info); CheckBruteForceSearchParam(field, search_info);
auto sub_qr = BruteForceSearch( auto sub_qr = BruteForceSearch(dataset,
dataset, vec_data, row_count, search_info.search_params_, bitset); vec_data,
row_count,
search_info.search_params_,
bitset,
data_type);
result.distances_ = std::move(sub_qr.mutable_distances()); result.distances_ = std::move(sub_qr.mutable_distances());
result.seg_offsets_ = std::move(sub_qr.mutable_seg_offsets()); result.seg_offsets_ = std::move(sub_qr.mutable_seg_offsets());

View File

@ -27,6 +27,9 @@ class ExecPlanNodeVisitor : public PlanNodeVisitor {
void void
visit(BinaryVectorANNS& node) override; visit(BinaryVectorANNS& node) override;
void
visit(Float16VectorANNS& node) override;
void void
visit(RetrievePlanNode& node) override; visit(RetrievePlanNode& node) override;

View File

@ -24,6 +24,9 @@ class ExtractInfoPlanNodeVisitor : public PlanNodeVisitor {
void void
visit(BinaryVectorANNS& node) override; visit(BinaryVectorANNS& node) override;
void
visit(Float16VectorANNS& node) override;
void void
visit(RetrievePlanNode& node) override; visit(RetrievePlanNode& node) override;

View File

@ -25,6 +25,11 @@ BinaryVectorANNS::accept(PlanNodeVisitor& visitor) {
visitor.visit(*this); visitor.visit(*this);
} }
void
Float16VectorANNS::accept(PlanNodeVisitor& visitor) {
visitor.visit(*this);
}
void void
RetrievePlanNode::accept(PlanNodeVisitor& visitor) { RetrievePlanNode::accept(PlanNodeVisitor& visitor) {
visitor.visit(*this); visitor.visit(*this);

View File

@ -25,6 +25,9 @@ class PlanNodeVisitor {
virtual void virtual void
visit(BinaryVectorANNS&) = 0; visit(BinaryVectorANNS&) = 0;
virtual void
visit(Float16VectorANNS&) = 0;
virtual void virtual void
visit(RetrievePlanNode&) = 0; visit(RetrievePlanNode&) = 0;
}; };

View File

@ -28,6 +28,9 @@ class ShowPlanNodeVisitor : public PlanNodeVisitor {
void void
visit(BinaryVectorANNS& node) override; visit(BinaryVectorANNS& node) override;
void
visit(Float16VectorANNS& node) override;
void void
visit(RetrievePlanNode& node) override; visit(RetrievePlanNode& node) override;

View File

@ -27,6 +27,9 @@ class VerifyPlanNodeVisitor : public PlanNodeVisitor {
void void
visit(BinaryVectorANNS& node) override; visit(BinaryVectorANNS& node) override;
void
visit(Float16VectorANNS& node) override;
void void
visit(RetrievePlanNode& node) override; visit(RetrievePlanNode& node) override;

View File

@ -212,4 +212,9 @@ ExecPlanNodeVisitor::visit(BinaryVectorANNS& node) {
VectorVisitorImpl<BinaryVector>(node); VectorVisitorImpl<BinaryVector>(node);
} }
void
ExecPlanNodeVisitor::visit(Float16VectorANNS& node) {
VectorVisitorImpl<Float16Vector>(node);
}
} // namespace milvus::query } // namespace milvus::query

View File

@ -47,6 +47,15 @@ ExtractInfoPlanNodeVisitor::visit(BinaryVectorANNS& node) {
} }
} }
void
ExtractInfoPlanNodeVisitor::visit(Float16VectorANNS& node) {
plan_info_.add_involved_field(node.search_info_.field_id_);
if (node.predicate_.has_value()) {
ExtractInfoExprVisitor expr_visitor(plan_info_);
node.predicate_.value()->accept(expr_visitor);
}
}
void void
ExtractInfoPlanNodeVisitor::visit(RetrievePlanNode& node) { ExtractInfoPlanNodeVisitor::visit(RetrievePlanNode& node) {
// Assert(node.predicate_.has_value()); // Assert(node.predicate_.has_value());

View File

@ -96,6 +96,30 @@ ShowPlanNodeVisitor::visit(BinaryVectorANNS& node) {
ret_ = json_body; ret_ = json_body;
} }
void
ShowPlanNodeVisitor::visit(Float16VectorANNS& node) {
assert(!ret_);
auto& info = node.search_info_;
Json json_body{
{"node_type", "Float16VectorANNS"}, //
{"metric_type", info.metric_type_}, //
{"field_id_", info.field_id_.get()}, //
{"topk", info.topk_}, //
{"search_params", info.search_params_}, //
{"placeholder_tag", node.placeholder_tag_}, //
};
if (node.predicate_.has_value()) {
ShowExprVisitor expr_show;
AssertInfo(node.predicate_.value(),
"[ShowPlanNodeVisitor]Can't get value from node predict");
json_body["predicate"] =
expr_show.call_child(node.predicate_->operator*());
} else {
json_body["predicate"] = "None";
}
ret_ = json_body;
}
void void
ShowPlanNodeVisitor::visit(RetrievePlanNode& node) { ShowPlanNodeVisitor::visit(RetrievePlanNode& node) {
} }

View File

@ -67,6 +67,10 @@ void
VerifyPlanNodeVisitor::visit(BinaryVectorANNS&) { VerifyPlanNodeVisitor::visit(BinaryVectorANNS&) {
} }
void
VerifyPlanNodeVisitor::visit(Float16VectorANNS&) {
}
void void
VerifyPlanNodeVisitor::visit(RetrievePlanNode&) { VerifyPlanNodeVisitor::visit(RetrievePlanNode&) {
} }

View File

@ -30,6 +30,9 @@ VectorBase::set_data_raw(ssize_t element_offset,
} else if (field_meta.get_data_type() == DataType::VECTOR_BINARY) { } else if (field_meta.get_data_type() == DataType::VECTOR_BINARY) {
return set_data_raw( return set_data_raw(
element_offset, VEC_FIELD_DATA(data, binary), element_count); element_offset, VEC_FIELD_DATA(data, binary), element_count);
} else if (field_meta.get_data_type() == DataType::VECTOR_FLOAT16) {
return set_data_raw(
element_offset, VEC_FIELD_DATA(data, float16), element_count);
} else { } else {
PanicInfo("unsupported"); PanicInfo("unsupported");
} }

View File

@ -148,12 +148,14 @@ class ConcurrentVectorImpl : public VectorBase {
ConcurrentVectorImpl& ConcurrentVectorImpl&
operator=(const ConcurrentVectorImpl&) = delete; operator=(const ConcurrentVectorImpl&) = delete;
using TraitType = using TraitType = std::conditional_t<
std::conditional_t<is_scalar, is_scalar,
Type, Type,
std::conditional_t<std::is_same_v<Type, float>, std::conditional_t<std::is_same_v<Type, float>,
FloatVector, FloatVector,
BinaryVector>>; std::conditional_t<std::is_same_v<Type, float16>,
Float16Vector,
BinaryVector>>>;
public: public:
explicit ConcurrentVectorImpl(ssize_t dim, int64_t size_per_chunk) explicit ConcurrentVectorImpl(ssize_t dim, int64_t size_per_chunk)
@ -389,4 +391,14 @@ class ConcurrentVector<BinaryVector>
int64_t binary_dim_; int64_t binary_dim_;
}; };
template <>
class ConcurrentVector<Float16Vector>
: public ConcurrentVectorImpl<float16, false> {
public:
ConcurrentVector(int64_t dim, int64_t size_per_chunk)
: ConcurrentVectorImpl<float16, false>::ConcurrentVectorImpl(
dim, size_per_chunk) {
}
};
} // namespace milvus::segcore } // namespace milvus::segcore

View File

@ -237,6 +237,11 @@ CreateIndex(const FieldMeta& field_meta,
field_index_meta, field_index_meta,
segment_max_row_count, segment_max_row_count,
segcore_config); segcore_config);
} else if (field_meta.get_data_type() == DataType::VECTOR_FLOAT16) {
return std::make_unique<VectorFieldIndexing>(field_meta,
field_index_meta,
segment_max_row_count,
segcore_config);
} else { } else {
// TODO // TODO
PanicInfo("unsupported"); PanicInfo("unsupported");

View File

@ -280,6 +280,11 @@ struct InsertRecord {
this->append_field_data<BinaryVector>( this->append_field_data<BinaryVector>(
field_id, field_meta.get_dim(), size_per_chunk); field_id, field_meta.get_dim(), size_per_chunk);
continue; continue;
} else if (field_meta.get_data_type() ==
DataType::VECTOR_FLOAT16) {
this->append_field_data<Float16Vector>(
field_id, field_meta.get_dim(), size_per_chunk);
continue;
} else { } else {
PanicInfo("unsupported"); PanicInfo("unsupported");
} }

View File

@ -346,6 +346,13 @@ SegmentGrowingImpl::bulk_subscript(FieldId field_id,
seg_offsets, seg_offsets,
count, count,
output.data()); output.data());
} else if (field_meta.get_data_type() == DataType::VECTOR_FLOAT16) {
bulk_subscript_impl<Float16Vector>(field_id,
field_meta.get_sizeof(),
vec_ptr,
seg_offsets,
count,
output.data());
} else { } else {
PanicInfo("logical error"); PanicInfo("logical error");
} }

View File

@ -892,6 +892,7 @@ SegmentSealedImpl::bulk_subscript(FieldId field_id,
} }
case DataType::VECTOR_FLOAT: case DataType::VECTOR_FLOAT:
case DataType::VECTOR_FLOAT16:
case DataType::VECTOR_BINARY: { case DataType::VECTOR_BINARY: {
aligned_vector<char> output(field_meta.get_sizeof() * count); aligned_vector<char> output(field_meta.get_sizeof() * count);
bulk_subscript_impl(field_meta.get_sizeof(), bulk_subscript_impl(field_meta.get_sizeof(),

View File

@ -243,6 +243,12 @@ CreateVectorDataArray(int64_t count, const FieldMeta& field_meta) {
obj->resize(num_bytes); obj->resize(num_bytes);
break; break;
} }
case DataType::VECTOR_FLOAT16: {
auto length = count * dim;
auto obj = vector_array->mutable_float16_vector();
obj->resize(length * sizeof(float16));
break;
}
default: { default: {
PanicInfo("unsupported datatype"); PanicInfo("unsupported datatype");
} }
@ -358,6 +364,13 @@ CreateVectorDataArrayFrom(const void* data_raw,
obj->assign(data, num_bytes); obj->assign(data, num_bytes);
break; break;
} }
case DataType::VECTOR_FLOAT16: {
auto length = count * dim;
auto data = reinterpret_cast<const char*>(data_raw);
auto obj = vector_array->mutable_float16_vector();
obj->assign(data, length * sizeof(float16));
break;
}
default: { default: {
PanicInfo("unsupported datatype"); PanicInfo("unsupported datatype");
} }

View File

@ -131,6 +131,7 @@ FieldDataImpl<Type, is_scalar>::FillFieldData(
return FillFieldData(values.data(), element_count); return FillFieldData(values.data(), element_count);
} }
case DataType::VECTOR_FLOAT: case DataType::VECTOR_FLOAT:
case DataType::VECTOR_FLOAT16:
case DataType::VECTOR_BINARY: { case DataType::VECTOR_BINARY: {
auto array_info = auto array_info =
GetDataInfoFromArray<arrow::FixedSizeBinaryArray, GetDataInfoFromArray<arrow::FixedSizeBinaryArray,
@ -161,5 +162,6 @@ template class FieldDataImpl<Json, true>;
// vector data // vector data
template class FieldDataImpl<int8_t, false>; template class FieldDataImpl<int8_t, false>;
template class FieldDataImpl<float, false>; template class FieldDataImpl<float, false>;
template class FieldDataImpl<float16, false>;
} // namespace milvus::storage } // namespace milvus::storage

View File

@ -85,6 +85,17 @@ class FieldData<BinaryVector> : public FieldDataImpl<uint8_t, false> {
int64_t binary_dim_; int64_t binary_dim_;
}; };
template <>
class FieldData<Float16Vector> : public FieldDataImpl<float16, false> {
public:
explicit FieldData(int64_t dim,
DataType data_type,
int64_t buffered_num_rows = 0)
: FieldDataImpl<float16, false>::FieldDataImpl(
dim, data_type, buffered_num_rows) {
}
};
using FieldDataPtr = std::shared_ptr<FieldDataBase>; using FieldDataPtr = std::shared_ptr<FieldDataBase>;
using FieldDataChannel = Channel<storage::FieldDataPtr>; using FieldDataChannel = Channel<storage::FieldDataPtr>;
using FieldDataChannelPtr = std::shared_ptr<FieldDataChannel>; using FieldDataChannelPtr = std::shared_ptr<FieldDataChannel>;

View File

@ -121,6 +121,7 @@ AddPayloadToArrowBuilder(std::shared_ptr<arrow::ArrayBuilder> builder,
builder, double_data, length); builder, double_data, length);
break; break;
} }
case DataType::VECTOR_FLOAT16:
case DataType::VECTOR_BINARY: case DataType::VECTOR_BINARY:
case DataType::VECTOR_FLOAT: { case DataType::VECTOR_FLOAT: {
add_vector_payload(builder, const_cast<uint8_t*>(raw_data), length); add_vector_payload(builder, const_cast<uint8_t*>(raw_data), length);
@ -215,6 +216,11 @@ CreateArrowBuilder(DataType data_type, int dim) {
return std::make_shared<arrow::FixedSizeBinaryBuilder>( return std::make_shared<arrow::FixedSizeBinaryBuilder>(
arrow::fixed_size_binary(dim / 8)); arrow::fixed_size_binary(dim / 8));
} }
case DataType::VECTOR_FLOAT16: {
AssertInfo(dim > 0, "invalid dim value");
return std::make_shared<arrow::FixedSizeBinaryBuilder>(
arrow::fixed_size_binary(dim * sizeof(float16)));
}
default: { default: {
PanicInfo("unsupported vector data type"); PanicInfo("unsupported vector data type");
} }
@ -272,6 +278,11 @@ CreateArrowSchema(DataType data_type, int dim) {
return arrow::schema( return arrow::schema(
{arrow::field("val", arrow::fixed_size_binary(dim / 8))}); {arrow::field("val", arrow::fixed_size_binary(dim / 8))});
} }
case DataType::VECTOR_FLOAT16: {
AssertInfo(dim > 0, "invalid dim value");
return arrow::schema({arrow::field(
"val", arrow::fixed_size_binary(dim * sizeof(float16)))});
}
default: { default: {
PanicInfo("unsupported vector data type"); PanicInfo("unsupported vector data type");
} }
@ -288,6 +299,9 @@ GetDimensionFromFileMetaData(const parquet::ColumnDescriptor* schema,
case DataType::VECTOR_BINARY: { case DataType::VECTOR_BINARY: {
return schema->type_length() * 8; return schema->type_length() * 8;
} }
case DataType::VECTOR_FLOAT16: {
return schema->type_length() / sizeof(float16);
}
default: default:
PanicInfo("unsupported data type"); PanicInfo("unsupported data type");
} }
@ -560,6 +574,9 @@ CreateFieldData(const DataType& type, int64_t dim, int64_t total_num_rows) {
case DataType::VECTOR_BINARY: case DataType::VECTOR_BINARY:
return std::make_shared<FieldData<BinaryVector>>( return std::make_shared<FieldData<BinaryVector>>(
dim, type, total_num_rows); dim, type, total_num_rows);
case DataType::VECTOR_FLOAT16:
return std::make_shared<FieldData<Float16Vector>>(
dim, type, total_num_rows);
default: default:
throw NotSupportedDataTypeException( throw NotSupportedDataTypeException(
"CreateFieldData not support data type " + datatype_name(type)); "CreateFieldData not support data type " + datatype_name(type));

View File

@ -25,6 +25,7 @@ set(MILVUS_TEST_FILES
test_concurrent_vector.cpp test_concurrent_vector.cpp
test_c_api.cpp test_c_api.cpp
test_expr.cpp test_expr.cpp
test_float16.cpp
test_growing.cpp test_growing.cpp
test_growing_index.cpp test_growing_index.cpp
test_indexing.cpp test_indexing.cpp

View File

@ -311,7 +311,7 @@ TEST(CApiTest, CPlan) {
milvus::proto::plan::PlanNode plan_node; milvus::proto::plan::PlanNode plan_node;
auto vector_anns = plan_node.mutable_vector_anns(); auto vector_anns = plan_node.mutable_vector_anns();
vector_anns->set_is_binary(true); vector_anns->set_vector_type(milvus::proto::plan::VectorType::BinaryVector);
vector_anns->set_placeholder_tag("$0"); vector_anns->set_placeholder_tag("$0");
vector_anns->set_field_id(100); vector_anns->set_field_id(100);
auto query_info = vector_anns->mutable_query_info(); auto query_info = vector_anns->mutable_query_info();
@ -950,7 +950,7 @@ TEST(CApiTest, SearchTest) {
milvus::proto::plan::PlanNode plan_node; milvus::proto::plan::PlanNode plan_node;
auto vector_anns = plan_node.mutable_vector_anns(); auto vector_anns = plan_node.mutable_vector_anns();
vector_anns->set_is_binary(false); vector_anns->set_vector_type(milvus::proto::plan::VectorType::FloatVector);
vector_anns->set_placeholder_tag("$0"); vector_anns->set_placeholder_tag("$0");
vector_anns->set_field_id(100); vector_anns->set_field_id(100);
auto query_info = vector_anns->mutable_query_info(); auto query_info = vector_anns->mutable_query_info();
@ -1275,7 +1275,7 @@ TEST(CApiTest, ReudceNullResult) {
milvus::proto::plan::PlanNode plan_node; milvus::proto::plan::PlanNode plan_node;
auto vector_anns = plan_node.mutable_vector_anns(); auto vector_anns = plan_node.mutable_vector_anns();
vector_anns->set_is_binary(false); vector_anns->set_vector_type(milvus::proto::plan::VectorType::FloatVector);
vector_anns->set_placeholder_tag("$0"); vector_anns->set_placeholder_tag("$0");
vector_anns->set_field_id(100); vector_anns->set_field_id(100);
auto query_info = vector_anns->mutable_query_info(); auto query_info = vector_anns->mutable_query_info();
@ -1359,7 +1359,7 @@ TEST(CApiTest, ReduceRemoveDuplicates) {
milvus::proto::plan::PlanNode plan_node; milvus::proto::plan::PlanNode plan_node;
auto vector_anns = plan_node.mutable_vector_anns(); auto vector_anns = plan_node.mutable_vector_anns();
vector_anns->set_is_binary(false); vector_anns->set_vector_type(milvus::proto::plan::VectorType::FloatVector);
vector_anns->set_placeholder_tag("$0"); vector_anns->set_placeholder_tag("$0");
vector_anns->set_field_id(100); vector_anns->set_field_id(100);
auto query_info = vector_anns->mutable_query_info(); auto query_info = vector_anns->mutable_query_info();
@ -1703,7 +1703,7 @@ TEST(CApiTest, Indexing_Without_Predicate) {
milvus::proto::plan::PlanNode plan_node; milvus::proto::plan::PlanNode plan_node;
auto vector_anns = plan_node.mutable_vector_anns(); auto vector_anns = plan_node.mutable_vector_anns();
vector_anns->set_is_binary(false); vector_anns->set_vector_type(milvus::proto::plan::VectorType::FloatVector);
vector_anns->set_placeholder_tag("$0"); vector_anns->set_placeholder_tag("$0");
vector_anns->set_field_id(100); vector_anns->set_field_id(100);
auto query_info = vector_anns->mutable_query_info(); auto query_info = vector_anns->mutable_query_info();

View File

@ -302,6 +302,36 @@ TEST(storage, InsertDataBinaryVector) {
ASSERT_EQ(data, new_data); ASSERT_EQ(data, new_data);
} }
TEST(storage, InsertDataFloat16Vector) {
std::vector<float16> data = {1, 2, 3, 4, 5, 6, 7, 8};
int DIM = 2;
auto field_data =
milvus::storage::CreateFieldData(storage::DataType::VECTOR_FLOAT16, DIM);
field_data->FillFieldData(data.data(), data.size() / DIM);
storage::InsertData insert_data(field_data);
storage::FieldDataMeta field_data_meta{100, 101, 102, 103};
insert_data.SetFieldDataMeta(field_data_meta);
insert_data.SetTimestamps(0, 100);
auto serialized_bytes = insert_data.Serialize(storage::StorageType::Remote);
std::shared_ptr<uint8_t[]> serialized_data_ptr(serialized_bytes.data(),
[&](uint8_t*) {});
auto new_insert_data = storage::DeserializeFileData(
serialized_data_ptr, serialized_bytes.size());
ASSERT_EQ(new_insert_data->GetCodecType(), storage::InsertDataType);
ASSERT_EQ(new_insert_data->GetTimeRage(),
std::make_pair(Timestamp(0), Timestamp(100)));
auto new_payload = new_insert_data->GetFieldData();
ASSERT_EQ(new_payload->get_data_type(), storage::DataType::VECTOR_FLOAT16);
ASSERT_EQ(new_payload->get_num_rows(), data.size() / DIM);
std::vector<float16> new_data(data.size());
memcpy(new_data.data(),
new_payload->Data(),
new_payload->get_num_rows() * sizeof(float16) * DIM);
ASSERT_EQ(data, new_data);
}
TEST(storage, IndexData) { TEST(storage, IndexData) {
std::vector<uint8_t> data = {1, 2, 3, 4, 5, 6, 7, 8}; std::vector<uint8_t> data = {1, 2, 3, 4, 5, 6, 7, 8};
auto field_data = milvus::storage::CreateFieldData(storage::DataType::INT8); auto field_data = milvus::storage::CreateFieldData(storage::DataType::INT8);

View File

@ -0,0 +1,420 @@
// Copyright (C) 2019-2020 Zilliz. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software distributed under the License
// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
// or implied. See the License for the specific language governing permissions and limitations under the License
#include <gtest/gtest.h>
#include "common/LoadInfo.h"
#include "common/Types.h"
#include "index/IndexFactory.h"
#include "knowhere/comp/index_param.h"
#include "query/ExprImpl.h"
#include "segcore/Reduce.h"
#include "segcore/reduce_c.h"
#include "test_utils/PbHelper.h"
#include "test_utils/indexbuilder_test_utils.h"
#include "pb/schema.pb.h"
#include "pb/plan.pb.h"
#include "query/Expr.h"
#include "query/Plan.h"
#include "query/Utils.h"
#include "query/PlanImpl.h"
#include "query/PlanNode.h"
#include "query/PlanProto.h"
#include "query/SearchBruteForce.h"
#include "query/generated/ExecPlanNodeVisitor.h"
#include "query/generated/PlanNodeVisitor.h"
#include "query/generated/ExecExprVisitor.h"
#include "query/generated/ExprVisitor.h"
#include "query/generated/ShowPlanNodeVisitor.h"
#include "segcore/Collection.h"
#include "segcore/SegmentSealed.h"
#include "segcore/SegmentGrowing.h"
#include "segcore/SegmentGrowingImpl.h"
#include "test_utils/AssertUtils.h"
#include "test_utils/DataGen.h"
using namespace milvus::segcore;
using namespace milvus;
using namespace milvus::index;
using namespace knowhere;
using milvus::index::VectorIndex;
using milvus::segcore::LoadIndexInfo;
const int64_t ROW_COUNT = 100 * 1000;
TEST(Float16, Insert) {
using namespace milvus;
using namespace milvus::query;
using namespace milvus::segcore;
int64_t N = ROW_COUNT;
constexpr int64_t size_per_chunk = 32 * 1024;
auto schema = std::make_shared<Schema>();
auto float16_vec_fid = schema->AddDebugField(
"float16vec", DataType::VECTOR_FLOAT16, 32, knowhere::metric::L2);
auto i64_fid = schema->AddDebugField("counter", DataType::INT64);
schema->set_primary_field_id(i64_fid);
auto dataset = DataGen(schema, N);
// auto seg_conf = SegcoreConfig::default_config();
auto segment = CreateGrowingSegment(schema, empty_index_meta);
segment->PreInsert(N);
segment->Insert(0,
N,
dataset.row_ids_.data(),
dataset.timestamps_.data(),
dataset.raw_);
auto float16_ptr = dataset.get_col<float16>(float16_vec_fid);
SegmentInternalInterface& interface = *segment;
auto num_chunk = interface.num_chunk();
ASSERT_EQ(num_chunk, upper_div(N, size_per_chunk));
auto row_count = interface.get_row_count();
ASSERT_EQ(N, row_count);
for (auto chunk_id = 0; chunk_id < num_chunk; ++chunk_id) {
auto float16_span =
interface.chunk_data<milvus::Float16Vector>(float16_vec_fid, chunk_id);
auto begin = chunk_id * size_per_chunk;
auto end = std::min((chunk_id + 1) * size_per_chunk, N);
auto size_of_chunk = end - begin;
for (int i = 0; i < size_of_chunk; ++i) {
// std::cout << float16_span.data()[i] << " " << float16_ptr[i + begin * 32] << std::endl;
ASSERT_EQ(float16_span.data()[i], float16_ptr[i + begin * 32]);
}
}
}
TEST(Float16, ShowExecutor) {
using namespace milvus::query;
using namespace milvus::segcore;
using namespace milvus;
auto metric_type = knowhere::metric::L2;
auto node = std::make_unique<Float16VectorANNS>();
auto schema = std::make_shared<Schema>();
auto field_id = schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT16, 16, metric_type);
int64_t num_queries = 100L;
auto raw_data = DataGen(schema, num_queries);
auto& info = node->search_info_;
info.metric_type_ = metric_type;
info.topk_ = 20;
info.field_id_ = field_id;
node->predicate_ = std::nullopt;
ShowPlanNodeVisitor show_visitor;
PlanNodePtr base(node.release());
auto res = show_visitor.call_child(*base);
auto dup = res;
std::cout << dup.dump(4);
}
TEST(Float16, ExecWithoutPredicateFlat) {
using namespace milvus::query;
using namespace milvus::segcore;
using namespace milvus;
auto schema = std::make_shared<Schema>();
auto vec_fid = schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT16, 32, knowhere::metric::L2);
schema->AddDebugField("age", DataType::FLOAT);
auto i64_fid = schema->AddDebugField("counter", DataType::INT64);
schema->set_primary_field_id(i64_fid);
const char* raw_plan = R"(vector_anns: <
field_id: 100
query_info: <
topk: 5
round_decimal: 3
metric_type: "L2"
search_params: "{\"nprobe\": 10}"
>
placeholder_tag: "$0"
>)";
auto plan_str = translate_text_plan_to_binary_plan(raw_plan);
auto plan =
CreateSearchPlanByExpr(*schema, plan_str.data(), plan_str.size());
int64_t N = ROW_COUNT;
auto dataset = DataGen(schema, N);
auto segment = CreateGrowingSegment(schema, empty_index_meta);
segment->PreInsert(N);
segment->Insert(0,
N,
dataset.row_ids_.data(),
dataset.timestamps_.data(),
dataset.raw_);
auto vec_ptr = dataset.get_col<float16>(vec_fid);
auto num_queries = 5;
auto ph_group_raw = CreateFloat16PlaceholderGroup(
num_queries, 32, 1024);
auto ph_group =
ParsePlaceholderGroup(plan.get(), ph_group_raw.SerializeAsString());
auto sr = segment->Search(plan.get(), ph_group.get());
int topk = 5;
query::Json json = SearchResultToJson(*sr);
std::cout << json.dump(2);
}
TEST(Float16, GetVector) {
auto metricType = knowhere::metric::L2;
auto schema = std::make_shared<Schema>();
auto pk = schema->AddDebugField("pk", DataType::INT64);
auto random = schema->AddDebugField("random", DataType::DOUBLE);
auto vec = schema->AddDebugField(
"embeddings", DataType::VECTOR_FLOAT16, 128, metricType);
schema->set_primary_field_id(pk);
std::map<std::string, std::string> index_params = {
{"index_type", "IVF_FLAT"},
{"metric_type", metricType},
{"nlist", "128"}};
std::map<std::string, std::string> type_params = {{"dim", "128"}};
FieldIndexMeta fieldIndexMeta(
vec, std::move(index_params), std::move(type_params));
auto& config = SegcoreConfig::default_config();
config.set_chunk_rows(1024);
config.set_enable_growing_segment_index(true);
std::map<FieldId, FieldIndexMeta> filedMap = {{vec, fieldIndexMeta}};
IndexMetaPtr metaPtr =
std::make_shared<CollectionIndexMeta>(100000, std::move(filedMap));
auto segment_growing = CreateGrowingSegment(schema, metaPtr);
auto segment = dynamic_cast<SegmentGrowingImpl*>(segment_growing.get());
int64_t per_batch = 5000;
int64_t n_batch = 20;
int64_t dim = 128;
for (int64_t i = 0; i < n_batch; i++) {
auto dataset = DataGen(schema, per_batch);
auto fakevec = dataset.get_col<float16>(vec);
auto offset = segment->PreInsert(per_batch);
segment->Insert(offset,
per_batch,
dataset.row_ids_.data(),
dataset.timestamps_.data(),
dataset.raw_);
auto num_inserted = (i + 1) * per_batch;
auto ids_ds = GenRandomIds(num_inserted);
auto result =
segment->bulk_subscript(vec, ids_ds->GetIds(), num_inserted);
auto vector = result.get()->mutable_vectors()->float16_vector();
EXPECT_TRUE(vector.size() == num_inserted * dim * sizeof(float16));
// EXPECT_TRUE(vector.size() == num_inserted * dim);
// for (size_t i = 0; i < num_inserted; ++i) {
// auto id = ids_ds->GetIds()[i];
// for (size_t j = 0; j < 128; ++j) {
// EXPECT_TRUE(vector[i * dim + j] ==
// fakevec[(id % per_batch) * dim + j]);
// }
// }
}
}
std::string
generate_collection_schema(std::string metric_type, int dim, bool is_fp16) {
namespace schema = milvus::proto::schema;
schema::CollectionSchema collection_schema;
collection_schema.set_name("collection_test");
auto vec_field_schema = collection_schema.add_fields();
vec_field_schema->set_name("fakevec");
vec_field_schema->set_fieldid(100);
if (is_fp16) {
vec_field_schema->set_data_type(schema::DataType::Float16Vector);
} else {
vec_field_schema->set_data_type(schema::DataType::FloatVector);
}
auto metric_type_param = vec_field_schema->add_index_params();
metric_type_param->set_key("metric_type");
metric_type_param->set_value(metric_type);
auto dim_param = vec_field_schema->add_type_params();
dim_param->set_key("dim");
dim_param->set_value(std::to_string(dim));
auto other_field_schema = collection_schema.add_fields();
other_field_schema->set_name("counter");
other_field_schema->set_fieldid(101);
other_field_schema->set_data_type(schema::DataType::Int64);
other_field_schema->set_is_primary_key(true);
auto other_field_schema2 = collection_schema.add_fields();
other_field_schema2->set_name("doubleField");
other_field_schema2->set_fieldid(102);
other_field_schema2->set_data_type(schema::DataType::Double);
std::string schema_string;
auto marshal = google::protobuf::TextFormat::PrintToString(
collection_schema, &schema_string);
assert(marshal);
return schema_string;
}
CCollection
NewCollection(const char* schema_proto_blob) {
auto proto = std::string(schema_proto_blob);
auto collection = std::make_unique<milvus::segcore::Collection>(proto);
return (void*)collection.release();
}
TEST(Float16, CApiCPlan) {
std::string schema_string =
generate_collection_schema(knowhere::metric::L2, 16, true);
auto collection = NewCollection(schema_string.c_str());
// const char* dsl_string = R"(
// {
// "bool": {
// "vector": {
// "fakevec": {
// "metric_type": "L2",
// "params": {
// "nprobe": 10
// },
// "query": "$0",
// "topk": 10,
// "round_decimal": 3
// }
// }
// }
// })";
milvus::proto::plan::PlanNode plan_node;
auto vector_anns = plan_node.mutable_vector_anns();
vector_anns->set_vector_type(milvus::proto::plan::VectorType::Float16Vector);
vector_anns->set_placeholder_tag("$0");
vector_anns->set_field_id(100);
auto query_info = vector_anns->mutable_query_info();
query_info->set_topk(10);
query_info->set_round_decimal(3);
query_info->set_metric_type("L2");
query_info->set_search_params(R"({"nprobe": 10})");
auto plan_str = plan_node.SerializeAsString();
void* plan = nullptr;
auto status = CreateSearchPlanByExpr(
collection, plan_str.data(), plan_str.size(), &plan);
ASSERT_EQ(status.error_code, Success);
int64_t field_id = -1;
status = GetFieldID(plan, &field_id);
ASSERT_EQ(status.error_code, Success);
auto col = static_cast<Collection*>(collection);
for (auto& [target_field_id, field_meta] :
col->get_schema()->get_fields()) {
if (field_meta.is_vector()) {
ASSERT_EQ(field_id, target_field_id.get());
}
}
ASSERT_NE(field_id, -1);
DeleteSearchPlan(plan);
DeleteCollection(collection);
}
TEST(Float16, RetrieveEmpty) {
auto schema = std::make_shared<Schema>();
auto fid_64 = schema->AddDebugField("i64", DataType::INT64);
auto DIM = 16;
auto fid_vec = schema->AddDebugField(
"vector_64", DataType::VECTOR_FLOAT16, DIM, knowhere::metric::L2);
schema->set_primary_field_id(fid_64);
int64_t N = 100;
int64_t req_size = 10;
auto choose = [=](int i) { return i * 3 % N; };
auto segment = CreateSealedSegment(schema);
auto plan = std::make_unique<query::RetrievePlan>(*schema);
std::vector<int64_t> values;
for (int i = 0; i < req_size; ++i) {
values.emplace_back(choose(i));
}
auto term_expr = std::make_unique<query::TermExprImpl<int64_t>>(
milvus::query::ColumnInfo(
fid_64, DataType::INT64, std::vector<std::string>()),
values,
proto::plan::GenericValue::kInt64Val);
plan->plan_node_ = std::make_unique<query::RetrievePlanNode>();
plan->plan_node_->predicate_ = std::move(term_expr);
std::vector<FieldId> target_offsets{fid_64, fid_vec};
plan->field_ids_ = target_offsets;
auto retrieve_results =
segment->Retrieve(plan.get(), 100, DEFAULT_MAX_OUTPUT_SIZE);
Assert(retrieve_results->fields_data_size() == target_offsets.size());
auto field0 = retrieve_results->fields_data(0);
auto field1 = retrieve_results->fields_data(1);
Assert(field0.has_scalars());
auto field0_data = field0.scalars().long_data();
Assert(field0_data.data_size() == 0);
Assert(field1.vectors().float16_vector().size() == 0);
}
TEST(Float16, ExecWithPredicate) {
using namespace milvus::query;
using namespace milvus::segcore;
auto schema = std::make_shared<Schema>();
schema->AddDebugField(
"fakevec", DataType::VECTOR_FLOAT16, 16, knowhere::metric::L2);
schema->AddDebugField("age", DataType::FLOAT);
auto i64_fid = schema->AddDebugField("counter", DataType::INT64);
schema->set_primary_field_id(i64_fid);
const char* raw_plan = R"(vector_anns: <
field_id: 100
predicates: <
binary_range_expr: <
column_info: <
field_id: 101
data_type: Float
>
lower_inclusive: true,
upper_inclusive: false,
lower_value: <
float_val: -1
>
upper_value: <
float_val: 1
>
>
>
query_info: <
topk: 5
round_decimal: 3
metric_type: "L2"
search_params: "{\"nprobe\": 10}"
>
placeholder_tag: "$0"
>)";
int64_t N = ROW_COUNT;
auto dataset = DataGen(schema, N);
auto segment = CreateGrowingSegment(schema, empty_index_meta);
segment->PreInsert(N);
segment->Insert(0,
N,
dataset.row_ids_.data(),
dataset.timestamps_.data(),
dataset.raw_);
auto plan_str = translate_text_plan_to_binary_plan(raw_plan);
auto plan =
CreateSearchPlanByExpr(*schema, plan_str.data(), plan_str.size());
auto num_queries = 5;
auto ph_group_raw = CreateFloat16PlaceholderGroup(num_queries, 16, 1024);
auto ph_group =
ParsePlaceholderGroup(plan.get(), ph_group_raw.SerializeAsString());
auto sr = segment->Search(plan.get(), ph_group.get());
int topk = 5;
query::Json json = SearchResultToJson(*sr);
std::cout << json.dump(2);
}

View File

@ -46,7 +46,7 @@ TEST(GrowingIndex, Correctness) {
milvus::proto::plan::PlanNode plan_node; milvus::proto::plan::PlanNode plan_node;
auto vector_anns = plan_node.mutable_vector_anns(); auto vector_anns = plan_node.mutable_vector_anns();
vector_anns->set_is_binary(false); vector_anns->set_vector_type(milvus::proto::plan::VectorType::FloatVector);
vector_anns->set_placeholder_tag("$0"); vector_anns->set_placeholder_tag("$0");
vector_anns->set_field_id(102); vector_anns->set_field_id(102);
auto query_info = vector_anns->mutable_query_info(); auto query_info = vector_anns->mutable_query_info();
@ -58,7 +58,7 @@ TEST(GrowingIndex, Correctness) {
milvus::proto::plan::PlanNode range_query_plan_node; milvus::proto::plan::PlanNode range_query_plan_node;
auto vector_range_querys = range_query_plan_node.mutable_vector_anns(); auto vector_range_querys = range_query_plan_node.mutable_vector_anns();
vector_range_querys->set_is_binary(false); vector_range_querys->set_vector_type(milvus::proto::plan::VectorType::FloatVector);
vector_range_querys->set_placeholder_tag("$0"); vector_range_querys->set_placeholder_tag("$0");
vector_range_querys->set_field_id(102); vector_range_querys->set_field_id(102);
auto range_query_info = vector_range_querys->mutable_query_info(); auto range_query_info = vector_range_querys->mutable_query_info();

View File

@ -73,12 +73,12 @@ GenQueryInfo(int64_t topk,
auto auto
GenAnns(proto::plan::Expr* predicate, GenAnns(proto::plan::Expr* predicate,
bool is_binary, proto::plan::VectorType vectorType,
int64_t field_id, int64_t field_id,
std::string placeholder_tag = "$0") { std::string placeholder_tag = "$0") {
auto query_info = GenQueryInfo(10, "L2", "{\"nprobe\": 10}", -1); auto query_info = GenQueryInfo(10, "L2", "{\"nprobe\": 10}", -1);
auto anns = new proto::plan::VectorANNS(); auto anns = new proto::plan::VectorANNS();
anns->set_is_binary(is_binary); anns->set_vector_type(vectorType);
anns->set_field_id(field_id); anns->set_field_id(field_id);
anns->set_allocated_predicates(predicate); anns->set_allocated_predicates(predicate);
anns->set_allocated_query_info(query_info); anns->set_allocated_query_info(query_info);
@ -177,8 +177,17 @@ GenTermPlan(const FieldMeta& fvec_meta,
auto expr = GenExpr().release(); auto expr = GenExpr().release();
expr->set_allocated_term_expr(term_expr); expr->set_allocated_term_expr(term_expr);
proto::plan::VectorType vector_type;
if (fvec_meta.get_data_type() == DataType::VECTOR_FLOAT) {
vector_type = proto::plan::VectorType::FloatVector;
} else if (fvec_meta.get_data_type() == DataType::VECTOR_BINARY) {
vector_type = proto::plan::VectorType::BinaryVector;
} else if (fvec_meta.get_data_type() == DataType::VECTOR_FLOAT16) {
vector_type = proto::plan::VectorType::Float16Vector;
}
auto anns = GenAnns(expr, auto anns = GenAnns(expr,
fvec_meta.get_data_type() == DataType::VECTOR_BINARY, vector_type,
fvec_meta.get_id().get(), fvec_meta.get_id().get(),
"$0"); "$0");
@ -215,8 +224,16 @@ GenAlwaysTrueExpr(const FieldMeta& fvec_meta, const FieldMeta& str_meta) {
auto auto
GenAlwaysFalsePlan(const FieldMeta& fvec_meta, const FieldMeta& str_meta) { GenAlwaysFalsePlan(const FieldMeta& fvec_meta, const FieldMeta& str_meta) {
auto always_false_expr = GenAlwaysFalseExpr(fvec_meta, str_meta); auto always_false_expr = GenAlwaysFalseExpr(fvec_meta, str_meta);
proto::plan::VectorType vector_type;
if (fvec_meta.get_data_type() == DataType::VECTOR_FLOAT) {
vector_type = proto::plan::VectorType::FloatVector;
} else if (fvec_meta.get_data_type() == DataType::VECTOR_BINARY) {
vector_type = proto::plan::VectorType::BinaryVector;
} else if (fvec_meta.get_data_type() == DataType::VECTOR_FLOAT16) {
vector_type = proto::plan::VectorType::Float16Vector;
}
auto anns = GenAnns(always_false_expr, auto anns = GenAnns(always_false_expr,
fvec_meta.get_data_type() == DataType::VECTOR_BINARY, vector_type,
fvec_meta.get_id().get(), fvec_meta.get_id().get(),
"$0"); "$0");
@ -228,8 +245,16 @@ GenAlwaysFalsePlan(const FieldMeta& fvec_meta, const FieldMeta& str_meta) {
auto auto
GenAlwaysTruePlan(const FieldMeta& fvec_meta, const FieldMeta& str_meta) { GenAlwaysTruePlan(const FieldMeta& fvec_meta, const FieldMeta& str_meta) {
auto always_true_expr = GenAlwaysTrueExpr(fvec_meta, str_meta); auto always_true_expr = GenAlwaysTrueExpr(fvec_meta, str_meta);
proto::plan::VectorType vector_type;
if (fvec_meta.get_data_type() == DataType::VECTOR_FLOAT) {
vector_type = proto::plan::VectorType::FloatVector;
} else if (fvec_meta.get_data_type() == DataType::VECTOR_BINARY) {
vector_type = proto::plan::VectorType::BinaryVector;
} else if (fvec_meta.get_data_type() == DataType::VECTOR_FLOAT16) {
vector_type = proto::plan::VectorType::Float16Vector;
}
auto anns = GenAnns(always_true_expr, auto anns = GenAnns(always_true_expr,
fvec_meta.get_data_type() == DataType::VECTOR_BINARY, vector_type,
fvec_meta.get_id().get(), fvec_meta.get_id().get(),
"$0"); "$0");
@ -353,9 +378,17 @@ TEST(StringExpr, Compare) {
auto expr = GenExpr().release(); auto expr = GenExpr().release();
expr->set_allocated_compare_expr(compare_expr); expr->set_allocated_compare_expr(compare_expr);
proto::plan::VectorType vector_type;
if (fvec_meta.get_data_type() == DataType::VECTOR_FLOAT) {
vector_type = proto::plan::VectorType::FloatVector;
} else if (fvec_meta.get_data_type() == DataType::VECTOR_BINARY) {
vector_type = proto::plan::VectorType::BinaryVector;
} else if (fvec_meta.get_data_type() == DataType::VECTOR_FLOAT16) {
vector_type = proto::plan::VectorType::Float16Vector;
}
auto anns = auto anns =
GenAnns(expr, GenAnns(expr,
fvec_meta.get_data_type() == DataType::VECTOR_BINARY, vector_type,
fvec_meta.get_id().get(), fvec_meta.get_id().get(),
"$0"); "$0");
@ -456,9 +489,17 @@ TEST(StringExpr, UnaryRange) {
auto expr = GenExpr().release(); auto expr = GenExpr().release();
expr->set_allocated_unary_range_expr(unary_range_expr); expr->set_allocated_unary_range_expr(unary_range_expr);
proto::plan::VectorType vector_type;
if (fvec_meta.get_data_type() == DataType::VECTOR_FLOAT) {
vector_type = proto::plan::VectorType::FloatVector;
} else if (fvec_meta.get_data_type() == DataType::VECTOR_BINARY) {
vector_type = proto::plan::VectorType::BinaryVector;
} else if (fvec_meta.get_data_type() == DataType::VECTOR_FLOAT16) {
vector_type = proto::plan::VectorType::Float16Vector;
}
auto anns = auto anns =
GenAnns(expr, GenAnns(expr,
fvec_meta.get_data_type() == DataType::VECTOR_BINARY, vector_type,
fvec_meta.get_id().get(), fvec_meta.get_id().get(),
"$0"); "$0");
@ -551,9 +592,17 @@ TEST(StringExpr, BinaryRange) {
auto expr = GenExpr().release(); auto expr = GenExpr().release();
expr->set_allocated_binary_range_expr(binary_range_expr); expr->set_allocated_binary_range_expr(binary_range_expr);
proto::plan::VectorType vector_type;
if (fvec_meta.get_data_type() == DataType::VECTOR_FLOAT) {
vector_type = proto::plan::VectorType::FloatVector;
} else if (fvec_meta.get_data_type() == DataType::VECTOR_BINARY) {
vector_type = proto::plan::VectorType::BinaryVector;
} else if (fvec_meta.get_data_type() == DataType::VECTOR_FLOAT16) {
vector_type = proto::plan::VectorType::Float16Vector;
}
auto anns = auto anns =
GenAnns(expr, GenAnns(expr,
fvec_meta.get_data_type() == DataType::VECTOR_BINARY, vector_type,
fvec_meta.get_id().get(), fvec_meta.get_id().get(),
"$0"); "$0");

View File

@ -105,6 +105,16 @@ struct GeneratedData {
auto src_data = reinterpret_cast<const T*>( auto src_data = reinterpret_cast<const T*>(
target_field_data.vectors().binary_vector().data()); target_field_data.vectors().binary_vector().data());
std::copy_n(src_data, len, ret.data()); std::copy_n(src_data, len, ret.data());
} else if (field_meta.get_data_type() ==
DataType::VECTOR_FLOAT16) {
// int len = raw_->num_rows() * field_meta.get_dim() * sizeof(float16);
int len = raw_->num_rows() * field_meta.get_dim();
ret.resize(len);
auto src_data =
reinterpret_cast<const T*>(target_field_data.vectors()
.float16_vector()
.data());
std::copy_n(src_data, len, ret.data());
} else { } else {
PanicInfo("unsupported"); PanicInfo("unsupported");
} }
@ -258,6 +268,15 @@ DataGen(SchemaPtr schema,
} }
insert_cols(data, N, field_meta); insert_cols(data, N, field_meta);
break; break;
}
case DataType::VECTOR_FLOAT16: {
auto dim = field_meta.get_dim();
vector<float16> final(dim * N);
for (auto& x : final) {
x = float16(distr(er) + offset);
}
insert_cols(final, N, field_meta);
break;
} }
case DataType::BOOL: { case DataType::BOOL: {
FixedVector<bool> data(N); FixedVector<bool> data(N);
@ -549,6 +568,27 @@ CreateBinaryPlaceholderGroupFromBlob(int64_t num_queries,
return raw_group; return raw_group;
} }
inline auto
CreateFloat16PlaceholderGroup(int64_t num_queries,
int64_t dim,
int64_t seed = 42) {
namespace ser = milvus::proto::common;
ser::PlaceholderGroup raw_group;
auto value = raw_group.add_placeholders();
value->set_tag("$0");
value->set_type(ser::PlaceholderType::Float16Vector);
std::normal_distribution<double> dis(0, 1);
std::default_random_engine e(seed);
for (int i = 0; i < num_queries; ++i) {
std::vector<float16> vec;
for (int d = 0; d < dim; ++d) {
vec.push_back(float16(dis(e)));
}
value->add_values(vec.data(), vec.size() * sizeof(float16));
}
return raw_group;
}
inline auto inline auto
SearchResultToVector(const SearchResult& sr) { SearchResultToVector(const SearchResult& sr) {
int64_t num_queries = sr.total_nq_; int64_t num_queries = sr.total_nq_;

View File

@ -104,7 +104,8 @@ func FilterInIndexedSegments(handler Handler, mt *meta, segments ...*SegmentInfo
} }
for _, field := range coll.Schema.GetFields() { for _, field := range coll.Schema.GetFields() {
if field.GetDataType() == schemapb.DataType_BinaryVector || if field.GetDataType() == schemapb.DataType_BinaryVector ||
field.GetDataType() == schemapb.DataType_FloatVector { field.GetDataType() == schemapb.DataType_FloatVector ||
field.GetDataType() == schemapb.DataType_Float16Vector {
vecFieldID[collection] = field.GetFieldID() vecFieldID[collection] = field.GetFieldID()
break break
} }

View File

@ -881,6 +881,22 @@ func interface2FieldData(schemaDataType schemapb.DataType, content []interface{}
data.Dim = len(data.Data) / int(numRows) data.Dim = len(data.Data) / int(numRows)
rst = data rst = data
case schemapb.DataType_Float16Vector:
var data = &storage.Float16VectorFieldData{
Data: []byte{},
}
for _, c := range content {
r, ok := c.([]byte)
if !ok {
return nil, errTransferType
}
data.Data = append(data.Data, r...)
}
data.Dim = len(data.Data) / 2 / int(numRows)
rst = data
case schemapb.DataType_BinaryVector: case schemapb.DataType_BinaryVector:
var data = &storage.BinaryVectorFieldData{ var data = &storage.BinaryVectorFieldData{
Data: []byte{}, Data: []byte{},

View File

@ -108,6 +108,7 @@ func TestCompactionTaskInnerMethods(t *testing.T) {
{true, schemapb.DataType_JSON, []interface{}{[]byte("{\"key\":\"value\"}"), []byte("{\"hello\":\"world\"}")}, "valid json"}, {true, schemapb.DataType_JSON, []interface{}{[]byte("{\"key\":\"value\"}"), []byte("{\"hello\":\"world\"}")}, "valid json"},
{true, schemapb.DataType_FloatVector, []interface{}{[]float32{1.0, 2.0}}, "valid floatvector"}, {true, schemapb.DataType_FloatVector, []interface{}{[]float32{1.0, 2.0}}, "valid floatvector"},
{true, schemapb.DataType_BinaryVector, []interface{}{[]byte{255}}, "valid binaryvector"}, {true, schemapb.DataType_BinaryVector, []interface{}{[]byte{255}}, "valid binaryvector"},
{true, schemapb.DataType_Float16Vector, []interface{}{[]byte{255, 255, 255, 255}}, "valid float16vector"},
{false, schemapb.DataType_Bool, []interface{}{1, 2}, "invalid bool"}, {false, schemapb.DataType_Bool, []interface{}{1, 2}, "invalid bool"},
{false, schemapb.DataType_Int8, []interface{}{nil, nil}, "invalid int8"}, {false, schemapb.DataType_Int8, []interface{}{nil, nil}, "invalid int8"},
{false, schemapb.DataType_Int16, []interface{}{nil, nil}, "invalid int16"}, {false, schemapb.DataType_Int16, []interface{}{nil, nil}, "invalid int16"},
@ -119,6 +120,7 @@ func TestCompactionTaskInnerMethods(t *testing.T) {
{false, schemapb.DataType_JSON, []interface{}{nil, nil}, "invalid json"}, {false, schemapb.DataType_JSON, []interface{}{nil, nil}, "invalid json"},
{false, schemapb.DataType_FloatVector, []interface{}{nil, nil}, "invalid floatvector"}, {false, schemapb.DataType_FloatVector, []interface{}{nil, nil}, "invalid floatvector"},
{false, schemapb.DataType_BinaryVector, []interface{}{nil, nil}, "invalid binaryvector"}, {false, schemapb.DataType_BinaryVector, []interface{}{nil, nil}, "invalid binaryvector"},
{false, schemapb.DataType_Float16Vector, []interface{}{nil, nil}, "invalid float16vector"},
{false, schemapb.DataType_None, nil, "invalid data type"}, {false, schemapb.DataType_None, nil, "invalid data type"},
} }

View File

@ -31,6 +31,8 @@ func estimateFieldDataSize(dim int64, numRows int64, dataType schemapb.DataType)
if dataType == schemapb.DataType_BinaryVector { if dataType == schemapb.DataType_BinaryVector {
return uint64(dim) / 8 * uint64(numRows), nil return uint64(dim) / 8 * uint64(numRows), nil
} }
if dataType == schemapb.DataType_Float16Vector {
return uint64(dim) * uint64(numRows) * 2, nil
}
return 0, nil return 0, nil
} }

View File

@ -37,7 +37,7 @@ func CheckQueryInfoIdentical(info1, info2 *planpb.QueryInfo) bool {
} }
func CheckVectorANNSIdentical(node1, node2 *planpb.VectorANNS) bool { func CheckVectorANNSIdentical(node1, node2 *planpb.VectorANNS) bool {
if node1.GetIsBinary() != node2.GetIsBinary() { if node1.GetVectorType() != node2.GetVectorType() {
return false return false
} }
if node1.GetFieldId() != node2.GetFieldId() { if node1.GetFieldId() != node2.GetFieldId() {

View File

@ -94,35 +94,35 @@ func TestCheckVectorANNSIdentical(t *testing.T) {
}{ }{
{ {
args: args{ args: args{
node1: &planpb.VectorANNS{IsBinary: true}, node1: &planpb.VectorANNS{VectorType: planpb.VectorType_BinaryVector},
node2: &planpb.VectorANNS{IsBinary: false}, node2: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector},
}, },
want: false, want: false,
}, },
{ {
args: args{ args: args{
node1: &planpb.VectorANNS{IsBinary: false, FieldId: 100}, node1: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector, FieldId: 100},
node2: &planpb.VectorANNS{IsBinary: false, FieldId: 101}, node2: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector, FieldId: 101},
}, },
want: false, want: false,
}, },
{ {
args: args{ args: args{
node1: &planpb.VectorANNS{IsBinary: false, FieldId: 100, PlaceholderTag: "$0"}, node1: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector, FieldId: 100, PlaceholderTag: "$0"},
node2: &planpb.VectorANNS{IsBinary: false, FieldId: 100, PlaceholderTag: "$1"}, node2: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector, FieldId: 100, PlaceholderTag: "$1"},
}, },
want: false, want: false,
}, },
{ {
args: args{ args: args{
node1: &planpb.VectorANNS{IsBinary: false, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 100}}, node1: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 100}},
node2: &planpb.VectorANNS{IsBinary: false, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 10}}, node2: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 10}},
}, },
want: false, want: false,
}, },
{ {
args: args{ args: args{
node1: &planpb.VectorANNS{IsBinary: false, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 1, MetricType: "L2", SearchParams: `{"nprobe": 10}`, RoundDecimal: 6}, node1: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 1, MetricType: "L2", SearchParams: `{"nprobe": 10}`, RoundDecimal: 6},
Predicates: &planpb.Expr{ Predicates: &planpb.Expr{
Expr: &planpb.Expr_ColumnExpr{ Expr: &planpb.Expr_ColumnExpr{
ColumnExpr: &planpb.ColumnExpr{ ColumnExpr: &planpb.ColumnExpr{
@ -130,7 +130,7 @@ func TestCheckVectorANNSIdentical(t *testing.T) {
}, },
}, },
}}, }},
node2: &planpb.VectorANNS{IsBinary: false, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 1, MetricType: "L2", SearchParams: `{"nprobe": 10}`, RoundDecimal: 6}, node2: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 1, MetricType: "L2", SearchParams: `{"nprobe": 10}`, RoundDecimal: 6},
Predicates: &planpb.Expr{ Predicates: &planpb.Expr{
Expr: &planpb.Expr_ValueExpr{ Expr: &planpb.Expr_ValueExpr{
ValueExpr: &planpb.ValueExpr{Value: NewInt(100)}, ValueExpr: &planpb.ValueExpr{Value: NewInt(100)},
@ -141,13 +141,13 @@ func TestCheckVectorANNSIdentical(t *testing.T) {
}, },
{ {
args: args{ args: args{
node1: &planpb.VectorANNS{IsBinary: false, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 1, MetricType: "L2", SearchParams: `{"nprobe": 10}`, RoundDecimal: 6}, node1: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 1, MetricType: "L2", SearchParams: `{"nprobe": 10}`, RoundDecimal: 6},
Predicates: &planpb.Expr{ Predicates: &planpb.Expr{
Expr: &planpb.Expr_ValueExpr{ Expr: &planpb.Expr_ValueExpr{
ValueExpr: &planpb.ValueExpr{Value: NewInt(100)}, ValueExpr: &planpb.ValueExpr{Value: NewInt(100)},
}, },
}}, }},
node2: &planpb.VectorANNS{IsBinary: false, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 1, MetricType: "L2", SearchParams: `{"nprobe": 10}`, RoundDecimal: 6}, node2: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 1, MetricType: "L2", SearchParams: `{"nprobe": 10}`, RoundDecimal: 6},
Predicates: &planpb.Expr{ Predicates: &planpb.Expr{
Expr: &planpb.Expr_ValueExpr{ Expr: &planpb.Expr_ValueExpr{
ValueExpr: &planpb.ValueExpr{Value: NewInt(100)}, ValueExpr: &planpb.ValueExpr{Value: NewInt(100)},
@ -194,7 +194,7 @@ func TestCheckPlanNodeIdentical(t *testing.T) {
node1: &planpb.PlanNode{ node1: &planpb.PlanNode{
Node: &planpb.PlanNode_VectorAnns{ Node: &planpb.PlanNode_VectorAnns{
VectorAnns: &planpb.VectorANNS{ VectorAnns: &planpb.VectorANNS{
IsBinary: true, VectorType: planpb.VectorType_BinaryVector,
}, },
}, },
OutputFieldIds: []int64{100}, OutputFieldIds: []int64{100},
@ -202,7 +202,7 @@ func TestCheckPlanNodeIdentical(t *testing.T) {
node2: &planpb.PlanNode{ node2: &planpb.PlanNode{
Node: &planpb.PlanNode_VectorAnns{ Node: &planpb.PlanNode_VectorAnns{
VectorAnns: &planpb.VectorANNS{ VectorAnns: &planpb.VectorANNS{
IsBinary: false, VectorType: planpb.VectorType_FloatVector,
}, },
}, },
OutputFieldIds: []int64{100}, OutputFieldIds: []int64{100},
@ -214,7 +214,7 @@ func TestCheckPlanNodeIdentical(t *testing.T) {
args: args{ args: args{
node1: &planpb.PlanNode{ node1: &planpb.PlanNode{
Node: &planpb.PlanNode_VectorAnns{ Node: &planpb.PlanNode_VectorAnns{
VectorAnns: &planpb.VectorANNS{IsBinary: false, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 1, MetricType: "L2", SearchParams: `{"nprobe": 10}`, RoundDecimal: 6}, VectorAnns: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 1, MetricType: "L2", SearchParams: `{"nprobe": 10}`, RoundDecimal: 6},
Predicates: &planpb.Expr{ Predicates: &planpb.Expr{
Expr: &planpb.Expr_ValueExpr{ Expr: &planpb.Expr_ValueExpr{
ValueExpr: &planpb.ValueExpr{Value: NewInt(100)}, ValueExpr: &planpb.ValueExpr{Value: NewInt(100)},
@ -225,7 +225,7 @@ func TestCheckPlanNodeIdentical(t *testing.T) {
}, },
node2: &planpb.PlanNode{ node2: &planpb.PlanNode{
Node: &planpb.PlanNode_VectorAnns{ Node: &planpb.PlanNode_VectorAnns{
VectorAnns: &planpb.VectorANNS{IsBinary: false, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 1, MetricType: "L2", SearchParams: `{"nprobe": 10}`, RoundDecimal: 6}, VectorAnns: &planpb.VectorANNS{VectorType: planpb.VectorType_FloatVector, FieldId: 100, PlaceholderTag: "$0", QueryInfo: &planpb.QueryInfo{Topk: 1, MetricType: "L2", SearchParams: `{"nprobe": 10}`, RoundDecimal: 6},
Predicates: &planpb.Expr{ Predicates: &planpb.Expr{
Expr: &planpb.Expr_ValueExpr{ Expr: &planpb.Expr_ValueExpr{
ValueExpr: &planpb.ValueExpr{Value: NewInt(100)}, ValueExpr: &planpb.ValueExpr{Value: NewInt(100)},

View File

@ -134,14 +134,22 @@ func CreateSearchPlan(schemaPb *schemapb.CollectionSchema, exprStr string, vecto
fieldID := vectorField.FieldID fieldID := vectorField.FieldID
dataType := vectorField.DataType dataType := vectorField.DataType
var vectorType planpb.VectorType
if !typeutil.IsVectorType(dataType) { if !typeutil.IsVectorType(dataType) {
return nil, fmt.Errorf("field (%s) to search is not of vector data type", vectorFieldName) return nil, fmt.Errorf("field (%s) to search is not of vector data type", vectorFieldName)
} else {
if dataType == schemapb.DataType_FloatVector {
vectorType = planpb.VectorType_FloatVector
} else if dataType == schemapb.DataType_BinaryVector {
vectorType = planpb.VectorType_BinaryVector
} else {
vectorType = planpb.VectorType_Float16Vector
}
} }
planNode := &planpb.PlanNode{ planNode := &planpb.PlanNode{
Node: &planpb.PlanNode_VectorAnns{ Node: &planpb.PlanNode_VectorAnns{
VectorAnns: &planpb.VectorANNS{ VectorAnns: &planpb.VectorANNS{
IsBinary: dataType == schemapb.DataType_BinaryVector, VectorType: vectorType,
Predicates: expr, Predicates: expr,
QueryInfo: queryInfo, QueryInfo: queryInfo,
PlaceholderTag: "$0", PlaceholderTag: "$0",

View File

@ -29,6 +29,12 @@ enum ArithOpType {
Mod = 5; Mod = 5;
}; };
enum VectorType {
BinaryVector = 0;
FloatVector = 1;
Float16Vector = 2;
};
message GenericValue { message GenericValue {
oneof val { oneof val {
bool bool_val = 1; bool bool_val = 1;
@ -176,7 +182,7 @@ message Expr {
} }
message VectorANNS { message VectorANNS {
bool is_binary = 1; VectorType vector_type = 1;
int64 field_id = 2; int64 field_id = 2;
Expr predicates = 3; Expr predicates = 3;
QueryInfo query_info = 4; QueryInfo query_info = 4;

View File

@ -116,6 +116,34 @@ func (ArithOpType) EnumDescriptor() ([]byte, []int) {
return fileDescriptor_2d655ab2f7683c23, []int{1} return fileDescriptor_2d655ab2f7683c23, []int{1}
} }
type VectorType int32
const (
VectorType_BinaryVector VectorType = 0
VectorType_FloatVector VectorType = 1
VectorType_Float16Vector VectorType = 2
)
var VectorType_name = map[int32]string{
0: "BinaryVector",
1: "FloatVector",
2: "Float16Vector",
}
var VectorType_value = map[string]int32{
"BinaryVector": 0,
"FloatVector": 1,
"Float16Vector": 2,
}
func (x VectorType) String() string {
return proto.EnumName(VectorType_name, int32(x))
}
func (VectorType) EnumDescriptor() ([]byte, []int) {
return fileDescriptor_2d655ab2f7683c23, []int{2}
}
// 0: invalid // 0: invalid
// 1: json_contains // 1: json_contains
// 2: json_contains_all // 2: json_contains_all
@ -206,7 +234,6 @@ func (BinaryExpr_BinaryOp) EnumDescriptor() ([]byte, []int) {
type GenericValue struct { type GenericValue struct {
// Types that are valid to be assigned to Val: // Types that are valid to be assigned to Val:
//
// *GenericValue_BoolVal // *GenericValue_BoolVal
// *GenericValue_Int64Val // *GenericValue_Int64Val
// *GenericValue_FloatVal // *GenericValue_FloatVal
@ -1251,7 +1278,6 @@ var xxx_messageInfo_AlwaysTrueExpr proto.InternalMessageInfo
type Expr struct { type Expr struct {
// Types that are valid to be assigned to Expr: // Types that are valid to be assigned to Expr:
//
// *Expr_TermExpr // *Expr_TermExpr
// *Expr_UnaryExpr // *Expr_UnaryExpr
// *Expr_BinaryExpr // *Expr_BinaryExpr
@ -1496,7 +1522,7 @@ func (*Expr) XXX_OneofWrappers() []interface{} {
} }
type VectorANNS struct { type VectorANNS struct {
IsBinary bool `protobuf:"varint,1,opt,name=is_binary,json=isBinary,proto3" json:"is_binary,omitempty"` VectorType VectorType `protobuf:"varint,1,opt,name=vector_type,json=vectorType,proto3,enum=milvus.proto.plan.VectorType" json:"vector_type,omitempty"`
FieldId int64 `protobuf:"varint,2,opt,name=field_id,json=fieldId,proto3" json:"field_id,omitempty"` FieldId int64 `protobuf:"varint,2,opt,name=field_id,json=fieldId,proto3" json:"field_id,omitempty"`
Predicates *Expr `protobuf:"bytes,3,opt,name=predicates,proto3" json:"predicates,omitempty"` Predicates *Expr `protobuf:"bytes,3,opt,name=predicates,proto3" json:"predicates,omitempty"`
QueryInfo *QueryInfo `protobuf:"bytes,4,opt,name=query_info,json=queryInfo,proto3" json:"query_info,omitempty"` QueryInfo *QueryInfo `protobuf:"bytes,4,opt,name=query_info,json=queryInfo,proto3" json:"query_info,omitempty"`
@ -1531,11 +1557,11 @@ func (m *VectorANNS) XXX_DiscardUnknown() {
var xxx_messageInfo_VectorANNS proto.InternalMessageInfo var xxx_messageInfo_VectorANNS proto.InternalMessageInfo
func (m *VectorANNS) GetIsBinary() bool { func (m *VectorANNS) GetVectorType() VectorType {
if m != nil { if m != nil {
return m.IsBinary return m.VectorType
} }
return false return VectorType_BinaryVector
} }
func (m *VectorANNS) GetFieldId() int64 { func (m *VectorANNS) GetFieldId() int64 {
@ -1623,7 +1649,6 @@ func (m *QueryPlanNode) GetLimit() int64 {
type PlanNode struct { type PlanNode struct {
// Types that are valid to be assigned to Node: // Types that are valid to be assigned to Node:
//
// *PlanNode_VectorAnns // *PlanNode_VectorAnns
// *PlanNode_Predicates // *PlanNode_Predicates
// *PlanNode_Query // *PlanNode_Query
@ -1728,6 +1753,7 @@ func (*PlanNode) XXX_OneofWrappers() []interface{} {
func init() { func init() {
proto.RegisterEnum("milvus.proto.plan.OpType", OpType_name, OpType_value) proto.RegisterEnum("milvus.proto.plan.OpType", OpType_name, OpType_value)
proto.RegisterEnum("milvus.proto.plan.ArithOpType", ArithOpType_name, ArithOpType_value) proto.RegisterEnum("milvus.proto.plan.ArithOpType", ArithOpType_name, ArithOpType_value)
proto.RegisterEnum("milvus.proto.plan.VectorType", VectorType_name, VectorType_value)
proto.RegisterEnum("milvus.proto.plan.JSONContainsExpr_JSONOp", JSONContainsExpr_JSONOp_name, JSONContainsExpr_JSONOp_value) proto.RegisterEnum("milvus.proto.plan.JSONContainsExpr_JSONOp", JSONContainsExpr_JSONOp_name, JSONContainsExpr_JSONOp_value)
proto.RegisterEnum("milvus.proto.plan.UnaryExpr_UnaryOp", UnaryExpr_UnaryOp_name, UnaryExpr_UnaryOp_value) proto.RegisterEnum("milvus.proto.plan.UnaryExpr_UnaryOp", UnaryExpr_UnaryOp_name, UnaryExpr_UnaryOp_value)
proto.RegisterEnum("milvus.proto.plan.BinaryExpr_BinaryOp", BinaryExpr_BinaryOp_name, BinaryExpr_BinaryOp_value) proto.RegisterEnum("milvus.proto.plan.BinaryExpr_BinaryOp", BinaryExpr_BinaryOp_name, BinaryExpr_BinaryOp_value)
@ -1758,116 +1784,118 @@ func init() {
func init() { proto.RegisterFile("plan.proto", fileDescriptor_2d655ab2f7683c23) } func init() { proto.RegisterFile("plan.proto", fileDescriptor_2d655ab2f7683c23) }
var fileDescriptor_2d655ab2f7683c23 = []byte{ var fileDescriptor_2d655ab2f7683c23 = []byte{
// 1762 bytes of a gzipped FileDescriptorProto // 1796 bytes of a gzipped FileDescriptorProto
0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x58, 0x4f, 0x93, 0xdb, 0x48, 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xc4, 0x58, 0x4f, 0x73, 0xdc, 0x48,
0x15, 0xb7, 0x2c, 0x7b, 0x2c, 0x3d, 0x79, 0x3c, 0x8a, 0x8a, 0x2a, 0x26, 0x09, 0x9b, 0x19, 0xb4, 0x15, 0x1f, 0x8d, 0xe6, 0x8f, 0xf4, 0x66, 0x3c, 0x56, 0x54, 0x54, 0xe1, 0x24, 0x6c, 0x6c, 0xb4,
0x5b, 0xec, 0x10, 0xc8, 0xa4, 0x36, 0xbb, 0x9b, 0xb0, 0xd9, 0x5a, 0x18, 0xcf, 0x9f, 0x8d, 0xcd, 0x5b, 0xac, 0x09, 0xc4, 0xa9, 0x64, 0x77, 0x13, 0x36, 0x5b, 0x0b, 0x1e, 0xff, 0x49, 0x66, 0xd8,
0x6e, 0x66, 0x06, 0xcd, 0x6c, 0x0e, 0x70, 0x50, 0xb5, 0xa5, 0x9e, 0x71, 0x13, 0xb9, 0xa5, 0x48, 0x8d, 0x6d, 0x64, 0x6f, 0x0e, 0x70, 0x50, 0xf5, 0x48, 0x6d, 0x4f, 0x13, 0x4d, 0x4b, 0x91, 0x5a,
0x2d, 0x27, 0xfe, 0x0a, 0xdc, 0xf8, 0x00, 0x9c, 0x38, 0x70, 0xe7, 0xc8, 0x05, 0xce, 0x14, 0x07, 0x93, 0xcc, 0x17, 0xe0, 0xc0, 0x8d, 0x0f, 0xc0, 0x89, 0x03, 0x77, 0x8e, 0x5c, 0xe0, 0x4c, 0x71,
0x8e, 0x1c, 0xa9, 0xe2, 0x13, 0x50, 0x7c, 0x01, 0xaa, 0x5f, 0x4b, 0x96, 0x3d, 0xd8, 0x19, 0x0f, 0xe0, 0xc8, 0x91, 0xaf, 0x40, 0xf1, 0x05, 0xa8, 0x7e, 0x2d, 0x8d, 0x66, 0xcc, 0x4c, 0x3c, 0x06,
0xa4, 0x8a, 0x5b, 0xeb, 0xf5, 0x7b, 0xbf, 0x7e, 0xff, 0xfa, 0xbd, 0xd7, 0x02, 0x48, 0x22, 0xc2, 0x57, 0x71, 0x6b, 0xbd, 0x7e, 0xef, 0xd7, 0xef, 0x5f, 0xbf, 0xf7, 0x5a, 0x00, 0x71, 0x48, 0xf8,
0x77, 0x93, 0x34, 0x16, 0xb1, 0x73, 0x6b, 0xc4, 0xa2, 0x71, 0x9e, 0xa9, 0xaf, 0x5d, 0xb9, 0x71, 0x4e, 0x9c, 0x44, 0x22, 0xb2, 0x6f, 0x8d, 0x58, 0x38, 0xce, 0x52, 0xf5, 0xb5, 0x23, 0x37, 0xee,
0xa7, 0x9d, 0x05, 0x43, 0x3a, 0x22, 0x8a, 0xe4, 0xfe, 0x59, 0x83, 0xf6, 0x33, 0xca, 0x69, 0xca, 0xb4, 0x53, 0x7f, 0x48, 0x47, 0x44, 0x91, 0x9c, 0xbf, 0x68, 0xd0, 0x7e, 0x41, 0x39, 0x4d, 0x98,
0x82, 0x17, 0x24, 0xca, 0xa9, 0x73, 0x17, 0x8c, 0x41, 0x1c, 0x47, 0xfe, 0x98, 0x44, 0x9b, 0xda, 0xff, 0x8a, 0x84, 0x19, 0xb5, 0xef, 0x82, 0x31, 0x88, 0xa2, 0xd0, 0x1b, 0x93, 0x70, 0x43, 0xdb,
0xb6, 0xb6, 0x63, 0xf4, 0x6a, 0x5e, 0x4b, 0x52, 0x5e, 0x90, 0xc8, 0x79, 0x0f, 0x4c, 0xc6, 0xc5, 0xd2, 0xb6, 0x8d, 0x5e, 0xc5, 0x6d, 0x4a, 0xca, 0x2b, 0x12, 0xda, 0x1f, 0x80, 0xc9, 0xb8, 0x78,
0xe3, 0x4f, 0x70, 0xb7, 0xbe, 0xad, 0xed, 0xe8, 0xbd, 0x9a, 0x67, 0x20, 0xa9, 0xd8, 0xbe, 0x88, 0xf2, 0x29, 0xee, 0x56, 0xb7, 0xb4, 0x6d, 0xbd, 0x57, 0x71, 0x0d, 0x24, 0xe5, 0xdb, 0xe7, 0x61,
0x62, 0x22, 0x70, 0x5b, 0xdf, 0xd6, 0x76, 0x34, 0xb9, 0x8d, 0x24, 0xb9, 0xbd, 0x05, 0x90, 0x89, 0x44, 0x04, 0x6e, 0xeb, 0x5b, 0xda, 0xb6, 0x26, 0xb7, 0x91, 0x24, 0xb7, 0x37, 0x01, 0x52, 0x91,
0x94, 0xf1, 0x4b, 0xdc, 0x6f, 0x6c, 0x6b, 0x3b, 0x66, 0xaf, 0xe6, 0x99, 0x8a, 0x26, 0x19, 0x9e, 0x30, 0x7e, 0x81, 0xfb, 0xb5, 0x2d, 0x6d, 0xdb, 0xec, 0x55, 0x5c, 0x53, 0xd1, 0x24, 0xc3, 0x53,
0x80, 0x49, 0xd2, 0x94, 0x4c, 0x70, 0xbf, 0xb9, 0xad, 0xed, 0x58, 0x8f, 0x36, 0x77, 0xff, 0xc3, 0x30, 0x49, 0x92, 0x90, 0x09, 0xee, 0xd7, 0xb7, 0xb4, 0xed, 0xd6, 0xe3, 0x8d, 0x9d, 0xff, 0xb0,
0x82, 0xdd, 0xae, 0xe4, 0x91, 0xc8, 0xc8, 0xfc, 0x82, 0x44, 0xfb, 0x4d, 0xd0, 0xc7, 0x24, 0x72, 0x60, 0xa7, 0x2b, 0x79, 0x24, 0x32, 0x32, 0xbf, 0x22, 0xe1, 0x5e, 0x1d, 0xf4, 0x31, 0x09, 0x9d,
0x7f, 0x01, 0x4d, 0xdc, 0x73, 0x3e, 0x85, 0x26, 0xee, 0x6d, 0x6a, 0xdb, 0xfa, 0x8e, 0xf5, 0x68, 0x5f, 0x40, 0x1d, 0xf7, 0xec, 0xcf, 0xa0, 0x8e, 0x7b, 0x1b, 0xda, 0x96, 0xbe, 0xdd, 0x7a, 0xbc,
0x6b, 0x01, 0xc8, 0xac, 0xd1, 0x9e, 0xe2, 0x76, 0xee, 0x82, 0x99, 0x91, 0x11, 0xf5, 0xc5, 0x24, 0xb9, 0x00, 0x64, 0xd6, 0x68, 0x57, 0x71, 0xdb, 0x77, 0xc1, 0x4c, 0xc9, 0x88, 0x7a, 0x62, 0x12,
0xa1, 0x68, 0x9e, 0xe1, 0x19, 0x92, 0x70, 0x3e, 0x49, 0xa8, 0xfb, 0x2b, 0x0d, 0xcc, 0x9f, 0xe5, 0x53, 0x34, 0xcf, 0x70, 0x0d, 0x49, 0x38, 0x9b, 0xc4, 0xd4, 0xf9, 0xb5, 0x06, 0xe6, 0xcf, 0x32,
0x34, 0x9d, 0xf4, 0xf9, 0x45, 0xec, 0x38, 0xd0, 0x10, 0x71, 0xf2, 0x12, 0x5d, 0xa4, 0x7b, 0xb8, 0x9a, 0x4c, 0xfa, 0xfc, 0x3c, 0xb2, 0x6d, 0xa8, 0x89, 0x28, 0x7e, 0x8d, 0x2e, 0xd2, 0x5d, 0x5c,
0x76, 0xb6, 0xc0, 0x1a, 0x51, 0x91, 0xb2, 0x40, 0x01, 0x48, 0x07, 0x98, 0x1e, 0x28, 0x92, 0x84, 0xdb, 0x9b, 0xd0, 0x1a, 0x51, 0x91, 0x30, 0x5f, 0x01, 0x48, 0x07, 0x98, 0x2e, 0x28, 0x92, 0x84,
0x70, 0xde, 0x87, 0xf5, 0x8c, 0x92, 0x34, 0x18, 0xfa, 0x09, 0x49, 0xc9, 0x28, 0x53, 0x3e, 0xf0, 0xb0, 0x3f, 0x84, 0xb5, 0x94, 0x92, 0xc4, 0x1f, 0x7a, 0x31, 0x49, 0xc8, 0x28, 0x55, 0x3e, 0x70,
0xda, 0x8a, 0x78, 0x8a, 0x34, 0xc9, 0x94, 0xc6, 0x39, 0x0f, 0xfd, 0x90, 0x06, 0x6c, 0x54, 0x38, 0xdb, 0x8a, 0x78, 0x82, 0x34, 0xc9, 0x94, 0x44, 0x19, 0x0f, 0xbc, 0x80, 0xfa, 0x6c, 0x94, 0x3b,
0x42, 0xf7, 0xda, 0x48, 0x3c, 0x54, 0x34, 0xf7, 0x9f, 0x1a, 0xc0, 0x41, 0x1c, 0xe5, 0x23, 0x8e, 0x42, 0x77, 0xdb, 0x48, 0x3c, 0x50, 0x34, 0xe7, 0x9f, 0x1a, 0xc0, 0x7e, 0x14, 0x66, 0x23, 0x8e,
0xda, 0xdc, 0x06, 0xe3, 0x82, 0xd1, 0x28, 0xf4, 0x59, 0x58, 0x68, 0xd4, 0xc2, 0xef, 0x7e, 0xe8, 0xda, 0xdc, 0x06, 0xe3, 0x9c, 0xd1, 0x30, 0xf0, 0x58, 0x90, 0x6b, 0xd4, 0xc4, 0xef, 0x7e, 0x60,
0x3c, 0x05, 0x33, 0x24, 0x82, 0x54, 0x36, 0x75, 0x1e, 0xbd, 0x37, 0xef, 0x8e, 0x22, 0x1f, 0x0e, 0x3f, 0x03, 0x33, 0x20, 0x82, 0x94, 0x36, 0x75, 0x1e, 0x7f, 0x30, 0xef, 0x8e, 0x3c, 0x1f, 0x0e,
0x89, 0x20, 0x52, 0x4b, 0xcf, 0x08, 0x8b, 0x95, 0xf3, 0x01, 0x74, 0x58, 0xe6, 0x27, 0x29, 0x1b, 0x88, 0x20, 0x52, 0x4b, 0xd7, 0x08, 0xf2, 0x95, 0xfd, 0x11, 0x74, 0x58, 0xea, 0xc5, 0x09, 0x1b,
0x91, 0x74, 0xe2, 0xbf, 0xa4, 0x13, 0xb4, 0xc9, 0xf0, 0xda, 0x2c, 0x3b, 0x55, 0xc4, 0xaf, 0x28, 0x91, 0x64, 0xe2, 0xbd, 0xa6, 0x13, 0xb4, 0xc9, 0x70, 0xdb, 0x2c, 0x3d, 0x51, 0xc4, 0xaf, 0x28,
0x7a, 0x8d, 0x65, 0x3e, 0xc9, 0x45, 0xdc, 0x3f, 0x44, 0x8b, 0x0c, 0xcf, 0x60, 0x59, 0x17, 0xbf, 0x7a, 0x8d, 0xa5, 0x1e, 0xc9, 0x44, 0xd4, 0x3f, 0x40, 0x8b, 0x0c, 0xd7, 0x60, 0x69, 0x17, 0xbf,
0xa5, 0x4f, 0x38, 0xcd, 0x04, 0x0d, 0xfd, 0x84, 0x88, 0xe1, 0x66, 0x73, 0x5b, 0x97, 0x3e, 0x51, 0xa5, 0x4f, 0x38, 0x4d, 0x05, 0x0d, 0xbc, 0x98, 0x88, 0xe1, 0x46, 0x7d, 0x4b, 0x97, 0x3e, 0x51,
0xa4, 0x53, 0x22, 0x86, 0xce, 0x0e, 0xd8, 0xf2, 0x0c, 0x92, 0x0a, 0x26, 0x58, 0xcc, 0xf1, 0x94, 0xa4, 0x13, 0x22, 0x86, 0xf6, 0x36, 0x58, 0xf2, 0x0c, 0x92, 0x08, 0x26, 0x58, 0xc4, 0xf1, 0x94,
0x35, 0x04, 0xe9, 0xb0, 0xec, 0xb4, 0x24, 0x7f, 0x45, 0x27, 0xee, 0x4f, 0x4a, 0x93, 0x8f, 0xde, 0x06, 0x82, 0x74, 0x58, 0x7a, 0x52, 0x90, 0xbf, 0xa2, 0x13, 0xe7, 0x27, 0x85, 0xc9, 0x87, 0xef,
0x24, 0xa9, 0xf3, 0x11, 0x34, 0x18, 0xbf, 0x88, 0xd1, 0x5c, 0xeb, 0xaa, 0x49, 0x18, 0xe1, 0xca, 0xe2, 0xc4, 0x7e, 0x04, 0x35, 0xc6, 0xcf, 0x23, 0x34, 0xb7, 0x75, 0xd9, 0x24, 0x8c, 0x70, 0xe9,
0x3f, 0x1e, 0xb2, 0x4a, 0x80, 0xa3, 0x37, 0x2c, 0x13, 0xd9, 0x7f, 0x0b, 0xb0, 0x0f, 0x26, 0xe6, 0x1f, 0x17, 0x59, 0x25, 0xc0, 0xe1, 0x3b, 0x96, 0x8a, 0xf4, 0xbf, 0x05, 0xd8, 0x03, 0x13, 0xf3,
0x0b, 0xca, 0x7f, 0x0a, 0xcd, 0xb1, 0xfc, 0x28, 0x00, 0xae, 0xcf, 0x31, 0xe4, 0x76, 0x7f, 0xaf, 0x05, 0xe5, 0x3f, 0x83, 0xfa, 0x58, 0x7e, 0xe4, 0x00, 0x57, 0xe7, 0x18, 0x72, 0x3b, 0x7f, 0xd0,
0x41, 0xe7, 0x1b, 0x4e, 0xd2, 0x89, 0x47, 0xf8, 0xa5, 0x42, 0xfa, 0x31, 0x58, 0x01, 0x1e, 0xe5, 0xa0, 0xf3, 0x0d, 0x27, 0xc9, 0xc4, 0x25, 0xfc, 0x42, 0x21, 0xfd, 0x18, 0x5a, 0x3e, 0x1e, 0xe5,
0xaf, 0xae, 0x10, 0x04, 0x55, 0xf4, 0xbf, 0x0f, 0xf5, 0x38, 0x29, 0x62, 0x7b, 0x7b, 0x81, 0xd8, 0xad, 0xae, 0x10, 0xf8, 0x65, 0xf4, 0xbf, 0x0f, 0xd5, 0x28, 0xce, 0x63, 0x7b, 0x7b, 0x81, 0xd8,
0x49, 0x82, 0x71, 0xad, 0xc7, 0x49, 0xa5, 0xb4, 0x7e, 0x23, 0xa5, 0x7f, 0x57, 0x87, 0x8d, 0x7d, 0x71, 0x8c, 0x71, 0xad, 0x46, 0x71, 0xa9, 0xb4, 0x7e, 0x2d, 0xa5, 0x7f, 0x5f, 0x85, 0xf5, 0x3d,
0xf6, 0x6e, 0xb5, 0xfe, 0x10, 0x36, 0xa2, 0xf8, 0x35, 0x4d, 0x7d, 0xc6, 0x83, 0x28, 0xcf, 0xd8, 0x76, 0xb3, 0x5a, 0x7f, 0x0c, 0xeb, 0x61, 0xf4, 0x96, 0x26, 0x1e, 0xe3, 0x7e, 0x98, 0xa5, 0x6c,
0xb8, 0xbc, 0x72, 0x1d, 0x24, 0xf7, 0x4b, 0xaa, 0x64, 0xcc, 0x93, 0x64, 0x8e, 0x51, 0xa5, 0x61, 0x5c, 0x5c, 0xb9, 0x0e, 0x92, 0xfb, 0x05, 0x55, 0x32, 0x66, 0x71, 0x3c, 0xc7, 0xa8, 0xd2, 0xb0,
0x07, 0xc9, 0x15, 0xe3, 0x1e, 0x58, 0x0a, 0x51, 0x99, 0xd8, 0x58, 0xcd, 0x44, 0x40, 0x19, 0x55, 0x83, 0xe4, 0x92, 0x71, 0x17, 0x5a, 0x0a, 0x51, 0x99, 0x58, 0x5b, 0xcd, 0x44, 0x40, 0x19, 0x55,
0xfc, 0xf6, 0xc0, 0x52, 0x47, 0x29, 0x84, 0xe6, 0x8a, 0x08, 0x28, 0x83, 0x6b, 0xf7, 0x2f, 0x1a, 0xfc, 0x76, 0xa1, 0xa5, 0x8e, 0x52, 0x08, 0xf5, 0x15, 0x11, 0x50, 0x06, 0xd7, 0xce, 0x5f, 0x35,
0x58, 0x07, 0xf1, 0x28, 0x21, 0xa9, 0xf2, 0xd2, 0x33, 0xb0, 0x23, 0x7a, 0x21, 0xfc, 0x1b, 0xbb, 0x68, 0xed, 0x47, 0xa3, 0x98, 0x24, 0xca, 0x4b, 0x2f, 0xc0, 0x0a, 0xe9, 0xb9, 0xf0, 0xae, 0xed,
0xaa, 0x23, 0xc5, 0x66, 0xae, 0x78, 0x1f, 0x6e, 0xa5, 0xec, 0x72, 0x38, 0x8f, 0x54, 0x5f, 0x05, 0xaa, 0x8e, 0x14, 0x9b, 0xb9, 0xe2, 0x7d, 0xb8, 0x95, 0xb0, 0x8b, 0xe1, 0x3c, 0x52, 0x75, 0x15,
0x69, 0x03, 0xe5, 0x0e, 0xae, 0xe6, 0x8b, 0xbe, 0x42, 0xbe, 0xb8, 0xbf, 0xd5, 0xc0, 0x38, 0xa7, 0xa4, 0x75, 0x94, 0xdb, 0xbf, 0x9c, 0x2f, 0xfa, 0x0a, 0xf9, 0xe2, 0xfc, 0x4e, 0x03, 0xe3, 0x8c,
0xe9, 0xe8, 0x9d, 0x44, 0xfc, 0x09, 0xac, 0xa1, 0x5f, 0xb3, 0xcd, 0xfa, 0x6a, 0x65, 0xb9, 0x60, 0x26, 0xa3, 0x1b, 0x89, 0xf8, 0x53, 0x68, 0xa0, 0x5f, 0xd3, 0x8d, 0xea, 0x6a, 0x65, 0x39, 0x67,
0x77, 0xee, 0x81, 0xc5, 0x32, 0x9f, 0x71, 0x1f, 0x8b, 0x5a, 0x11, 0x7d, 0x93, 0x65, 0x7d, 0xfe, 0xb7, 0xef, 0x41, 0x8b, 0xa5, 0x1e, 0xe3, 0x1e, 0x16, 0xb5, 0x3c, 0xfa, 0x26, 0x4b, 0xfb, 0xfc,
0xa5, 0x24, 0xb8, 0x7f, 0xaa, 0x83, 0xfd, 0xd3, 0xb3, 0x93, 0xe3, 0x83, 0x98, 0x0b, 0xc2, 0x78, 0xb9, 0x24, 0x38, 0x7f, 0xae, 0x82, 0xf5, 0xd3, 0xd3, 0xe3, 0xa3, 0xfd, 0x88, 0x0b, 0xc2, 0x78,
0xf6, 0x4e, 0xb4, 0xfd, 0x1c, 0x0c, 0x1a, 0xd1, 0x11, 0xe5, 0x62, 0x65, 0x7d, 0xa7, 0x02, 0xce, 0x7a, 0x23, 0xda, 0x7e, 0x01, 0x06, 0x0d, 0xe9, 0x88, 0x72, 0xb1, 0xb2, 0xbe, 0x53, 0x01, 0xfb,
0xd3, 0x19, 0x17, 0xdf, 0x5f, 0x20, 0x76, 0x55, 0x5b, 0x24, 0x9c, 0x24, 0x78, 0x47, 0x7f, 0x08, 0xd9, 0x8c, 0x8b, 0xef, 0x2f, 0x10, 0xbb, 0xac, 0x2d, 0x12, 0x8e, 0x63, 0xbc, 0xa3, 0x3f, 0x04,
0x4e, 0x89, 0xe3, 0x57, 0xed, 0x48, 0x15, 0x56, 0xbb, 0xdc, 0x39, 0x2b, 0xdb, 0xd2, 0x11, 0xac, 0xbb, 0xc0, 0xf1, 0xca, 0x76, 0xa4, 0x0a, 0xab, 0x55, 0xec, 0x9c, 0x16, 0x6d, 0xe9, 0x10, 0x1a,
0x29, 0x59, 0xc7, 0x82, 0x56, 0x9f, 0x8f, 0x49, 0xc4, 0x42, 0xbb, 0xe6, 0xb4, 0xc1, 0x28, 0xf1, 0x4a, 0xd6, 0x6e, 0x41, 0xb3, 0xcf, 0xc7, 0x24, 0x64, 0x81, 0x55, 0xb1, 0xdb, 0x60, 0x14, 0xf8,
0x6d, 0xcd, 0xd9, 0x90, 0x49, 0xa9, 0xbe, 0xba, 0x51, 0x64, 0xd7, 0xe7, 0x08, 0x7c, 0x62, 0xeb, 0x96, 0x66, 0xaf, 0xcb, 0xa4, 0x54, 0x5f, 0xdd, 0x30, 0xb4, 0xaa, 0x73, 0x04, 0x3e, 0xb1, 0x74,
0xee, 0xaf, 0x35, 0x30, 0xb1, 0x2c, 0xa1, 0xef, 0x3e, 0x41, 0xf5, 0x35, 0x54, 0xff, 0x83, 0x05, 0xe7, 0x37, 0x1a, 0x98, 0x58, 0x96, 0xd0, 0x77, 0x9f, 0xa2, 0xfa, 0x1a, 0xaa, 0xff, 0xd1, 0x02,
0xea, 0x4f, 0x39, 0xd5, 0xaa, 0x50, 0xfc, 0x01, 0x34, 0x83, 0x21, 0x8b, 0xc2, 0x22, 0x2d, 0xbf, 0xf5, 0xa7, 0x9c, 0x6a, 0x95, 0x2b, 0xfe, 0x00, 0xea, 0xfe, 0x90, 0x85, 0x41, 0x9e, 0x96, 0xdf,
0xbd, 0x40, 0x50, 0xca, 0x78, 0x8a, 0xcb, 0xdd, 0x82, 0x56, 0x21, 0x3d, 0xaf, 0x7a, 0x0b, 0xf4, 0x5e, 0x20, 0x28, 0x65, 0x5c, 0xc5, 0xe5, 0x6c, 0x42, 0x33, 0x97, 0x9e, 0x57, 0xbd, 0x09, 0xfa,
0xe3, 0x58, 0xd8, 0x9a, 0xfb, 0x37, 0x0d, 0x40, 0x55, 0x1d, 0x54, 0xea, 0xf1, 0x8c, 0x52, 0xdf, 0x51, 0x24, 0x2c, 0xcd, 0xf9, 0xbb, 0x06, 0xa0, 0xaa, 0x0e, 0x2a, 0xf5, 0x64, 0x46, 0xa9, 0xef,
0x5b, 0x80, 0x5d, 0xb1, 0x16, 0xcb, 0x42, 0xad, 0x1f, 0x40, 0x43, 0xde, 0xa5, 0xeb, 0xb4, 0x42, 0x2d, 0xc0, 0x2e, 0x59, 0xf3, 0x65, 0xae, 0xd6, 0x0f, 0xa0, 0x26, 0xef, 0xd2, 0x55, 0x5a, 0x21,
0x26, 0x69, 0x03, 0x5e, 0x97, 0xa2, 0x40, 0x2e, 0xb7, 0x01, 0xb9, 0xdc, 0xc7, 0x60, 0x94, 0x67, 0x93, 0xb4, 0x01, 0xaf, 0x4b, 0x5e, 0x20, 0x97, 0xdb, 0x80, 0x5c, 0xce, 0x13, 0x30, 0x8a, 0xb3,
0xcd, 0x1b, 0xd1, 0x01, 0xf8, 0x3a, 0xbe, 0x64, 0x01, 0x89, 0xba, 0x3c, 0xb4, 0x35, 0x67, 0x1d, 0xe6, 0x8d, 0xe8, 0x00, 0x7c, 0x1d, 0x5d, 0x30, 0x9f, 0x84, 0x5d, 0x1e, 0x58, 0x9a, 0xbd, 0x06,
0xcc, 0xe2, 0xfb, 0x24, 0xb5, 0xeb, 0xee, 0x5f, 0x35, 0x58, 0x57, 0x82, 0xdd, 0x94, 0x89, 0xe1, 0x66, 0xfe, 0x7d, 0x9c, 0x58, 0x55, 0xe7, 0x6f, 0x1a, 0xac, 0x29, 0xc1, 0x6e, 0xc2, 0xc4, 0xf0,
0x49, 0xf2, 0x3f, 0xa7, 0xeb, 0x67, 0x60, 0x10, 0x09, 0xe5, 0x4f, 0x5b, 0xc1, 0xbd, 0x85, 0xa3, 0x38, 0xfe, 0x9f, 0xd3, 0xf5, 0x73, 0x30, 0x88, 0x84, 0xf2, 0xa6, 0xad, 0xe0, 0xde, 0xc2, 0xd1,
0x13, 0x9e, 0x86, 0xf7, 0xbb, 0x45, 0x8a, 0xa3, 0x0f, 0x61, 0x5d, 0x95, 0x96, 0x38, 0xa1, 0x29, 0x09, 0x4f, 0xc3, 0xfb, 0xdd, 0x24, 0xf9, 0xd1, 0x07, 0xb0, 0xa6, 0x4a, 0x4b, 0x14, 0xd3, 0x84,
0xe1, 0xe1, 0xaa, 0xcd, 0xa1, 0x8d, 0x52, 0x27, 0x4a, 0xc8, 0xfd, 0x8d, 0x56, 0xf6, 0x08, 0x3c, 0xf0, 0x60, 0xd5, 0xe6, 0xd0, 0x46, 0xa9, 0x63, 0x25, 0xe4, 0xfc, 0x56, 0x2b, 0x7a, 0x04, 0x1e,
0x04, 0x43, 0x56, 0xba, 0x5e, 0xbb, 0x91, 0xeb, 0xeb, 0xab, 0xb8, 0xde, 0xd9, 0x9d, 0xb9, 0x62, 0x82, 0x21, 0x2b, 0x5c, 0xaf, 0x5d, 0xcb, 0xf5, 0xd5, 0x55, 0x5c, 0x6f, 0xef, 0xcc, 0x5c, 0xb1,
0xd7, 0x99, 0x2a, 0x4b, 0xd9, 0x1f, 0xeb, 0x70, 0x67, 0xce, 0xe5, 0x47, 0x63, 0x12, 0xbd, 0xbb, 0xab, 0x4c, 0x95, 0xa5, 0xec, 0x4f, 0x55, 0xb8, 0x33, 0xe7, 0xf2, 0xc3, 0x31, 0x09, 0x6f, 0xae,
0x76, 0xf6, 0xff, 0xf6, 0x7f, 0x51, 0xd5, 0x1b, 0x37, 0x9a, 0x02, 0x9a, 0x37, 0x9a, 0x02, 0x6c, 0x9d, 0xfd, 0xbf, 0xfd, 0x9f, 0x57, 0xf5, 0xda, 0xb5, 0xa6, 0x80, 0xfa, 0xb5, 0xa6, 0x00, 0x0b,
0xe8, 0x74, 0xa3, 0xd7, 0x64, 0x92, 0x9d, 0xa7, 0x6a, 0x06, 0x72, 0xff, 0xde, 0x82, 0x06, 0x7a, 0x3a, 0xdd, 0xf0, 0x2d, 0x99, 0xa4, 0x67, 0x89, 0x9a, 0x81, 0x9c, 0x7f, 0x34, 0xa1, 0x86, 0xde,
0xef, 0x29, 0x98, 0x82, 0xa6, 0x23, 0x9f, 0xbe, 0x49, 0xd2, 0xc2, 0x77, 0x77, 0x17, 0xa0, 0x96, 0x7b, 0x06, 0xa6, 0xa0, 0xc9, 0xc8, 0xa3, 0xef, 0xe2, 0x24, 0xf7, 0xdd, 0xdd, 0x05, 0xa8, 0x45,
0xad, 0x44, 0x0e, 0xef, 0xa2, 0x6c, 0x2b, 0x5f, 0x00, 0xe4, 0x32, 0x2c, 0x4a, 0x58, 0x05, 0xff, 0x2b, 0x91, 0xc3, 0xbb, 0x28, 0xda, 0xca, 0x97, 0x00, 0x99, 0x0c, 0x8b, 0x12, 0x56, 0xc1, 0xff,
0x3b, 0x6f, 0x2b, 0x3a, 0xf2, 0xd1, 0x90, 0x4f, 0xcb, 0xc2, 0x1e, 0x58, 0x03, 0x56, 0xc9, 0xeb, 0xce, 0xfb, 0x8a, 0x8e, 0x7c, 0x34, 0x64, 0xd3, 0xb2, 0xb0, 0x0b, 0xad, 0x01, 0x2b, 0xe5, 0xf5,
0x4b, 0x03, 0x57, 0xd5, 0x87, 0x5e, 0xcd, 0x83, 0x41, 0x55, 0x58, 0x0e, 0xa0, 0x1d, 0xa8, 0x96, 0xa5, 0x81, 0x2b, 0xeb, 0x43, 0xaf, 0xe2, 0xc2, 0xa0, 0x2c, 0x2c, 0xfb, 0xd0, 0xf6, 0x55, 0xcb,
0xad, 0x20, 0xd4, 0xe0, 0x70, 0x6f, 0x61, 0xec, 0xa7, 0x9d, 0xbd, 0x57, 0xf3, 0xac, 0x60, 0xa6, 0x56, 0x10, 0x6a, 0x70, 0xb8, 0xb7, 0x30, 0xf6, 0xd3, 0xce, 0xde, 0xab, 0xb8, 0x2d, 0x7f, 0xa6,
0xd1, 0x3f, 0x07, 0x5b, 0x59, 0x91, 0xca, 0x94, 0x52, 0x40, 0xca, 0xbd, 0xdf, 0x5d, 0x66, 0xcb, 0xd1, 0xbf, 0x04, 0x4b, 0x59, 0x91, 0xc8, 0x94, 0x52, 0x40, 0xca, 0xbd, 0xdf, 0x5d, 0x66, 0xcb,
0x34, 0xf9, 0x7a, 0x35, 0xaf, 0x93, 0xcf, 0x4f, 0x57, 0xa7, 0x70, 0xab, 0xb0, 0x6a, 0x06, 0x6f, 0x34, 0xf9, 0x7a, 0x15, 0xb7, 0x93, 0xcd, 0x4f, 0x57, 0x27, 0x70, 0x2b, 0xb7, 0x6a, 0x06, 0xaf,
0x0d, 0xf1, 0xdc, 0xa5, 0xb6, 0xcd, 0x02, 0x6e, 0x0c, 0xae, 0xcc, 0x6b, 0x02, 0xb6, 0x0a, 0xc4, 0x81, 0x78, 0xce, 0x52, 0xdb, 0x66, 0x01, 0xd7, 0x07, 0x97, 0xe6, 0x35, 0x01, 0x9b, 0x39, 0x62,
0x32, 0x4f, 0x7d, 0x3a, 0x26, 0xd1, 0x2c, 0x7e, 0x0b, 0xf1, 0x1f, 0x2c, 0xc5, 0x5f, 0x74, 0x71, 0x91, 0xa7, 0x1e, 0x1d, 0x93, 0x70, 0x16, 0xbf, 0x89, 0xf8, 0x0f, 0x96, 0xe2, 0x2f, 0xba, 0x38,
0x7a, 0x35, 0xef, 0xce, 0x60, 0xf9, 0xb5, 0xaa, 0xec, 0x50, 0xa7, 0xe2, 0x39, 0xc6, 0x35, 0x76, 0xbd, 0x8a, 0x7b, 0x67, 0xb0, 0xfc, 0x5a, 0x95, 0x76, 0xa8, 0x53, 0xf1, 0x1c, 0xe3, 0x0a, 0x3b,
0x4c, 0x0b, 0x48, 0x65, 0x47, 0x55, 0x53, 0xbe, 0x00, 0xc0, 0x74, 0x54, 0x50, 0xe6, 0xd2, 0x74, 0xa6, 0x05, 0xa4, 0xb4, 0xa3, 0xac, 0x29, 0x5f, 0x02, 0x60, 0x3a, 0x2a, 0x28, 0x73, 0x69, 0xba,
0x99, 0x4e, 0xea, 0x32, 0x5d, 0xc6, 0xd3, 0xb1, 0x7d, 0x6f, 0x7a, 0xcf, 0x51, 0x1e, 0xae, 0xb9, 0x4c, 0x27, 0x75, 0x99, 0x2e, 0xe3, 0xe9, 0xd8, 0xbe, 0x3b, 0xbd, 0xe7, 0x28, 0x0f, 0x57, 0xdc,
0xe7, 0x65, 0xba, 0x04, 0xd5, 0xcb, 0x63, 0x0f, 0x2c, 0x8a, 0xcf, 0x08, 0x85, 0x60, 0x2d, 0x45, 0xf3, 0x22, 0x5d, 0xfc, 0xf2, 0xe5, 0xb1, 0x0b, 0x2d, 0x8a, 0xcf, 0x08, 0x85, 0xd0, 0x5a, 0x8a,
0xa8, 0x1e, 0x1b, 0x12, 0x81, 0x56, 0x4f, 0x8f, 0xe7, 0x60, 0x13, 0xbc, 0x48, 0xbe, 0x48, 0x4b, 0x50, 0x3e, 0x36, 0x24, 0x02, 0x2d, 0x9f, 0x1e, 0x2f, 0xc1, 0x22, 0x78, 0x91, 0x3c, 0x91, 0x14,
0x43, 0xda, 0x4b, 0x73, 0x65, 0xfe, 0xce, 0xc9, 0x5c, 0x21, 0x73, 0x14, 0xe7, 0x0c, 0x9c, 0x5f, 0x86, 0xb4, 0x97, 0xe6, 0xca, 0xfc, 0x9d, 0x93, 0xb9, 0x42, 0xe6, 0x28, 0xf6, 0x29, 0xd8, 0xbf,
0x66, 0x31, 0xf7, 0x83, 0xa2, 0xa3, 0x2b, 0xc0, 0x75, 0x04, 0x7c, 0x7f, 0x85, 0xe1, 0xa3, 0x57, 0x4c, 0x23, 0xee, 0xf9, 0x79, 0x47, 0x57, 0x80, 0x6b, 0x08, 0xf8, 0xe1, 0x0a, 0xc3, 0x47, 0xaf,
0xf3, 0x6c, 0x09, 0x30, 0x4b, 0xdb, 0x5f, 0x83, 0x86, 0x84, 0x71, 0xff, 0xa1, 0x01, 0xbc, 0xa0, 0xe2, 0x5a, 0x12, 0x60, 0x96, 0xb6, 0xd7, 0x80, 0x9a, 0x84, 0x71, 0x7e, 0x55, 0x05, 0x78, 0x45,
0x81, 0x88, 0xd3, 0xee, 0xf1, 0xf1, 0x59, 0xf1, 0xd8, 0x53, 0x31, 0x51, 0xff, 0x07, 0xe4, 0x63, 0x7d, 0x11, 0x25, 0xdd, 0xa3, 0xa3, 0x53, 0x59, 0x26, 0xc7, 0xf8, 0xa5, 0xa6, 0x12, 0x6d, 0xd1,
0x4f, 0x85, 0x6d, 0xee, 0x19, 0x5a, 0x9f, 0x7f, 0x86, 0x3e, 0x01, 0x48, 0x52, 0x1a, 0xb2, 0x80, 0x83, 0x52, 0xb9, 0x1f, 0xb9, 0xb0, 0xe4, 0xc0, 0x78, 0xba, 0x9e, 0x7b, 0xa9, 0x56, 0xe7, 0x5f,
0x08, 0x9a, 0x5d, 0xd7, 0x5c, 0x67, 0x58, 0x9d, 0xcf, 0x01, 0x5e, 0xc9, 0x57, 0xb7, 0x2a, 0xcb, 0xaa, 0x4f, 0x01, 0xe2, 0x84, 0x06, 0xcc, 0x27, 0x82, 0xa6, 0x57, 0xf5, 0xdf, 0x19, 0x56, 0xfb,
0x8d, 0xa5, 0xe1, 0x9e, 0x3e, 0xcd, 0x3d, 0xf3, 0xd5, 0xf4, 0x95, 0xfe, 0x21, 0x6c, 0x24, 0x11, 0x0b, 0x80, 0x37, 0xf2, 0x61, 0xae, 0x2a, 0x77, 0x6d, 0x69, 0x46, 0x4c, 0x5f, 0xef, 0xae, 0xf9,
0x09, 0xe8, 0x30, 0x8e, 0x42, 0x9a, 0xfa, 0x82, 0x5c, 0xe2, 0x9d, 0x34, 0xbd, 0xce, 0x0c, 0xf9, 0x66, 0xfa, 0x90, 0xff, 0x18, 0xd6, 0xe3, 0x90, 0xf8, 0x74, 0x18, 0x85, 0x01, 0x4d, 0x3c, 0x41,
0x9c, 0x5c, 0xba, 0x13, 0x58, 0x47, 0x80, 0xd3, 0x88, 0xf0, 0xe3, 0x38, 0xa4, 0x57, 0xf4, 0xd5, 0x2e, 0xf0, 0xda, 0x9a, 0x6e, 0x67, 0x86, 0x7c, 0x46, 0x2e, 0x9c, 0x09, 0xac, 0x21, 0xc0, 0x49,
0x56, 0xd7, 0xf7, 0x36, 0x18, 0x2c, 0xf3, 0x83, 0x38, 0xe7, 0xa2, 0x78, 0xcf, 0xb4, 0x58, 0x76, 0x48, 0xf8, 0x51, 0x14, 0xd0, 0x4b, 0xfa, 0x6a, 0xab, 0xeb, 0x7b, 0x1b, 0x0c, 0x96, 0x7a, 0x7e,
0x20, 0x3f, 0x9d, 0x6f, 0x41, 0x33, 0x62, 0x23, 0xa6, 0x66, 0x0b, 0xdd, 0x53, 0x1f, 0xee, 0xbf, 0x94, 0x71, 0x91, 0x3f, 0x79, 0x9a, 0x2c, 0xdd, 0x97, 0x9f, 0xf6, 0xb7, 0xa0, 0x1e, 0xb2, 0x11,
0x34, 0x30, 0xa6, 0xc7, 0xee, 0x81, 0x35, 0x46, 0x67, 0xfb, 0x84, 0xf3, 0xec, 0x2d, 0x5d, 0xa8, 0x53, 0xe3, 0x87, 0xee, 0xaa, 0x0f, 0xe7, 0x5f, 0x1a, 0x18, 0xd3, 0x63, 0x77, 0xa7, 0x11, 0x20,
0x0a, 0x89, 0xcc, 0x2d, 0x25, 0xd3, 0xe5, 0x3c, 0x73, 0x3e, 0x9b, 0x53, 0xfc, 0xed, 0xad, 0x54, 0x9c, 0xa7, 0xef, 0x69, 0x54, 0x65, 0xd4, 0x64, 0xfa, 0x29, 0x99, 0x2e, 0xe7, 0xa9, 0xfd, 0xf9,
0x8a, 0xce, 0xa8, 0xfe, 0x23, 0x68, 0xa2, 0xeb, 0x0a, 0x2f, 0x6f, 0x2f, 0xf3, 0x72, 0xa9, 0x6d, 0x9c, 0xe2, 0xef, 0xef, 0xb6, 0x52, 0x74, 0x46, 0xf5, 0x1f, 0x41, 0x1d, 0x5d, 0x97, 0x7b, 0x79,
0xaf, 0xe6, 0x29, 0x01, 0xf9, 0x88, 0x8f, 0x73, 0x91, 0xe4, 0xc2, 0x2f, 0xe3, 0x2f, 0x63, 0xac, 0x6b, 0x99, 0x97, 0x0b, 0x6d, 0x7b, 0x15, 0x57, 0x09, 0xc8, 0x77, 0x7e, 0x94, 0x89, 0x38, 0x13,
0xef, 0xe8, 0x5e, 0x47, 0xd1, 0xbf, 0x54, 0x69, 0x90, 0xc9, 0xb4, 0xe2, 0x71, 0x48, 0xef, 0xff, 0x5e, 0x11, 0x7f, 0x19, 0x63, 0x7d, 0x5b, 0x77, 0x3b, 0x8a, 0xfe, 0x5c, 0xa5, 0x41, 0x2a, 0x33,
0x41, 0x83, 0x35, 0xd5, 0x91, 0xe6, 0xe7, 0xa6, 0x0d, 0xb0, 0x9e, 0xa5, 0x94, 0x08, 0x9a, 0x9e, 0x8f, 0x47, 0x01, 0xbd, 0xff, 0x47, 0x0d, 0x1a, 0xaa, 0x69, 0xcd, 0x8f, 0x56, 0xeb, 0xd0, 0x7a,
0x0f, 0x09, 0xb7, 0x35, 0xc7, 0x86, 0x76, 0x41, 0x38, 0x7a, 0x95, 0x13, 0x39, 0xbb, 0xb6, 0xc1, 0x91, 0x50, 0x22, 0x68, 0x72, 0x36, 0x24, 0xdc, 0xd2, 0x6c, 0x0b, 0xda, 0x39, 0xe1, 0xf0, 0x4d,
0xf8, 0x9a, 0x66, 0x19, 0xee, 0xeb, 0x38, 0x58, 0xd1, 0x2c, 0x53, 0x9b, 0x0d, 0xc7, 0x84, 0xa6, 0x46, 0xe4, 0x78, 0xdb, 0x06, 0xe3, 0x6b, 0x9a, 0xa6, 0xb8, 0xaf, 0xe3, 0xec, 0x45, 0xd3, 0x54,
0x5a, 0x36, 0x25, 0xdf, 0x71, 0x2c, 0xd4, 0xd7, 0x9a, 0x04, 0x3e, 0x4d, 0xe9, 0x05, 0x7b, 0xf3, 0x6d, 0xd6, 0x6c, 0x13, 0xea, 0x6a, 0x59, 0x97, 0x7c, 0x47, 0x91, 0x50, 0x5f, 0x0d, 0x09, 0x7c,
0x9c, 0x88, 0x60, 0x68, 0xb7, 0x24, 0xf0, 0x69, 0x9c, 0x89, 0x29, 0xc5, 0x90, 0xb2, 0x6a, 0x69, 0x92, 0xd0, 0x73, 0xf6, 0xee, 0x25, 0x11, 0xfe, 0xd0, 0x6a, 0x4a, 0xe0, 0x93, 0x28, 0x15, 0x53,
0xca, 0x25, 0xd6, 0x30, 0x1b, 0x9c, 0x35, 0xa8, 0xf7, 0xb9, 0x6d, 0x49, 0xd2, 0x71, 0x2c, 0xfa, 0x8a, 0x21, 0x65, 0xd5, 0xd2, 0x94, 0x4b, 0x2c, 0x73, 0x16, 0xd8, 0x0d, 0xa8, 0xf6, 0xb9, 0xd5,
0xdc, 0x6e, 0xdf, 0x7f, 0x06, 0xd6, 0x4c, 0x23, 0x97, 0x06, 0x7c, 0xc3, 0x5f, 0xf2, 0xf8, 0x35, 0x92, 0xa4, 0xa3, 0x48, 0xf4, 0xb9, 0xd5, 0xbe, 0xff, 0x02, 0x5a, 0x33, 0xbd, 0x5e, 0x1a, 0xf0,
0x57, 0xd3, 0x6b, 0x37, 0x94, 0x13, 0x5f, 0x0b, 0xf4, 0xb3, 0x7c, 0x60, 0xd7, 0xe5, 0xe2, 0x79, 0x0d, 0x7f, 0xcd, 0xa3, 0xb7, 0x5c, 0x0d, 0xb8, 0xdd, 0x40, 0x0e, 0x85, 0x4d, 0xd0, 0x4f, 0xb3,
0x1e, 0xd9, 0xba, 0x5c, 0x1c, 0xb2, 0xb1, 0xdd, 0x40, 0x4a, 0x1c, 0xda, 0xcd, 0xfd, 0x8f, 0x7f, 0x81, 0x55, 0x95, 0x8b, 0x97, 0x59, 0x68, 0xe9, 0x72, 0x71, 0xc0, 0xc6, 0x56, 0x0d, 0x29, 0x51,
0xfe, 0xd1, 0x25, 0x13, 0xc3, 0x7c, 0xb0, 0x1b, 0xc4, 0xa3, 0x87, 0xca, 0xdd, 0x0f, 0x58, 0x5c, 0x60, 0xd5, 0xef, 0xef, 0x15, 0xd7, 0x0f, 0x71, 0x2c, 0x68, 0xab, 0x12, 0xa9, 0x68, 0xca, 0x1b,
0xac, 0x1e, 0x32, 0x2e, 0x68, 0xca, 0x49, 0xf4, 0x10, 0x23, 0xf0, 0x50, 0x46, 0x20, 0x19, 0x0c, 0xcf, 0xf1, 0x07, 0x9b, 0x22, 0x68, 0xf6, 0x2d, 0x58, 0x43, 0xc2, 0xa3, 0x27, 0x39, 0xa9, 0xba,
0xd6, 0xf0, 0xeb, 0xe3, 0x7f, 0x07, 0x00, 0x00, 0xff, 0xff, 0x48, 0x4f, 0x55, 0x7b, 0xe8, 0x13, 0xf7, 0xc9, 0xcf, 0x1f, 0x5d, 0x30, 0x31, 0xcc, 0x06, 0x3b, 0x7e, 0x34, 0x7a, 0xa8, 0x42, 0xf6,
0x00, 0x00, 0x80, 0x45, 0xf9, 0xea, 0x21, 0xe3, 0x82, 0x26, 0x9c, 0x84, 0x0f, 0x31, 0x8a, 0x0f, 0x65, 0x14,
0xe3, 0xc1, 0xa0, 0x81, 0x5f, 0x9f, 0xfc, 0x3b, 0x00, 0x00, 0xff, 0xff, 0x3f, 0x1a, 0x78, 0xf1,
0x4f, 0x14, 0x00, 0x00,
} }

View File

@ -29,9 +29,8 @@ const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
type InvalidateCollMetaCacheRequest struct { type InvalidateCollMetaCacheRequest struct {
// MsgType: // MsgType:
// // DropCollection -> {meta cache, dml channels}
// DropCollection -> {meta cache, dml channels} // Other -> {meta cache}
// Other -> {meta cache}
Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"`
DbName string `protobuf:"bytes,2,opt,name=db_name,json=dbName,proto3" json:"db_name,omitempty"` DbName string `protobuf:"bytes,2,opt,name=db_name,json=dbName,proto3" json:"db_name,omitempty"`
CollectionName string `protobuf:"bytes,3,opt,name=collection_name,json=collectionName,proto3" json:"collection_name,omitempty"` CollectionName string `protobuf:"bytes,3,opt,name=collection_name,json=collectionName,proto3" json:"collection_name,omitempty"`

View File

@ -251,7 +251,7 @@ func (SyncType) EnumDescriptor() ([]byte, []int) {
return fileDescriptor_aab7cc9a69ed26e8, []int{6} return fileDescriptor_aab7cc9a69ed26e8, []int{6}
} }
// --------------------QueryCoord grpc request and response proto------------------ //--------------------QueryCoord grpc request and response proto------------------
type ShowCollectionsRequest struct { type ShowCollectionsRequest struct {
Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"`
// Not useful for now // Not useful for now
@ -1336,7 +1336,7 @@ func (m *SyncNewCreatedPartitionRequest) GetPartitionID() int64 {
return 0 return 0
} }
// -----------------query node grpc request and response proto---------------- //-----------------query node grpc request and response proto----------------
type LoadMetaInfo struct { type LoadMetaInfo struct {
LoadType LoadType `protobuf:"varint,1,opt,name=load_type,json=loadType,proto3,enum=milvus.proto.query.LoadType" json:"load_type,omitempty"` LoadType LoadType `protobuf:"varint,1,opt,name=load_type,json=loadType,proto3,enum=milvus.proto.query.LoadType" json:"load_type,omitempty"`
CollectionID int64 `protobuf:"varint,2,opt,name=collectionID,proto3" json:"collectionID,omitempty"` CollectionID int64 `protobuf:"varint,2,opt,name=collectionID,proto3" json:"collectionID,omitempty"`
@ -2488,7 +2488,7 @@ func (m *GetLoadInfoResponse) GetPartitions() []int64 {
return nil return nil
} }
// ----------------request auto triggered by QueryCoord----------------- //----------------request auto triggered by QueryCoord-----------------
type HandoffSegmentsRequest struct { type HandoffSegmentsRequest struct {
Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"`
SegmentInfos []*SegmentInfo `protobuf:"bytes,2,rep,name=segmentInfos,proto3" json:"segmentInfos,omitempty"` SegmentInfos []*SegmentInfo `protobuf:"bytes,2,rep,name=segmentInfos,proto3" json:"segmentInfos,omitempty"`
@ -3185,7 +3185,7 @@ func (m *UnsubscribeChannelInfo) GetCollectionChannels() []*UnsubscribeChannels
return nil return nil
} }
// ---- synchronize messages proto between QueryCoord and QueryNode ----- //---- synchronize messages proto between QueryCoord and QueryNode -----
type SegmentChangeInfo struct { type SegmentChangeInfo struct {
OnlineNodeID int64 `protobuf:"varint,1,opt,name=online_nodeID,json=onlineNodeID,proto3" json:"online_nodeID,omitempty"` OnlineNodeID int64 `protobuf:"varint,1,opt,name=online_nodeID,json=onlineNodeID,proto3" json:"online_nodeID,omitempty"`
OnlineSegments []*SegmentInfo `protobuf:"bytes,2,rep,name=online_segments,json=onlineSegments,proto3" json:"online_segments,omitempty"` OnlineSegments []*SegmentInfo `protobuf:"bytes,2,rep,name=online_segments,json=onlineSegments,proto3" json:"online_segments,omitempty"`

View File

@ -793,28 +793,28 @@ type RootCoordClient interface {
GetComponentStates(ctx context.Context, in *milvuspb.GetComponentStatesRequest, opts ...grpc.CallOption) (*milvuspb.ComponentStates, error) GetComponentStates(ctx context.Context, in *milvuspb.GetComponentStatesRequest, opts ...grpc.CallOption) (*milvuspb.ComponentStates, error)
GetTimeTickChannel(ctx context.Context, in *internalpb.GetTimeTickChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error) GetTimeTickChannel(ctx context.Context, in *internalpb.GetTimeTickChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error)
GetStatisticsChannel(ctx context.Context, in *internalpb.GetStatisticsChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error) GetStatisticsChannel(ctx context.Context, in *internalpb.GetStatisticsChannelRequest, opts ...grpc.CallOption) (*milvuspb.StringResponse, error)
// * //*
// @brief This method is used to create collection // @brief This method is used to create collection
// //
// @param CreateCollectionRequest, use to provide collection information to be created. // @param CreateCollectionRequest, use to provide collection information to be created.
// //
// @return Status // @return Status
CreateCollection(ctx context.Context, in *milvuspb.CreateCollectionRequest, opts ...grpc.CallOption) (*commonpb.Status, error) CreateCollection(ctx context.Context, in *milvuspb.CreateCollectionRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
// * //*
// @brief This method is used to delete collection. // @brief This method is used to delete collection.
// //
// @param DropCollectionRequest, collection name is going to be deleted. // @param DropCollectionRequest, collection name is going to be deleted.
// //
// @return Status // @return Status
DropCollection(ctx context.Context, in *milvuspb.DropCollectionRequest, opts ...grpc.CallOption) (*commonpb.Status, error) DropCollection(ctx context.Context, in *milvuspb.DropCollectionRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
// * //*
// @brief This method is used to test collection existence. // @brief This method is used to test collection existence.
// //
// @param HasCollectionRequest, collection name is going to be tested. // @param HasCollectionRequest, collection name is going to be tested.
// //
// @return BoolResponse // @return BoolResponse
HasCollection(ctx context.Context, in *milvuspb.HasCollectionRequest, opts ...grpc.CallOption) (*milvuspb.BoolResponse, error) HasCollection(ctx context.Context, in *milvuspb.HasCollectionRequest, opts ...grpc.CallOption) (*milvuspb.BoolResponse, error)
// * //*
// @brief This method is used to get collection schema. // @brief This method is used to get collection schema.
// //
// @param DescribeCollectionRequest, target collection name. // @param DescribeCollectionRequest, target collection name.
@ -825,28 +825,28 @@ type RootCoordClient interface {
CreateAlias(ctx context.Context, in *milvuspb.CreateAliasRequest, opts ...grpc.CallOption) (*commonpb.Status, error) CreateAlias(ctx context.Context, in *milvuspb.CreateAliasRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
DropAlias(ctx context.Context, in *milvuspb.DropAliasRequest, opts ...grpc.CallOption) (*commonpb.Status, error) DropAlias(ctx context.Context, in *milvuspb.DropAliasRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
AlterAlias(ctx context.Context, in *milvuspb.AlterAliasRequest, opts ...grpc.CallOption) (*commonpb.Status, error) AlterAlias(ctx context.Context, in *milvuspb.AlterAliasRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
// * //*
// @brief This method is used to list all collections. // @brief This method is used to list all collections.
// //
// @return StringListResponse, collection name list // @return StringListResponse, collection name list
ShowCollections(ctx context.Context, in *milvuspb.ShowCollectionsRequest, opts ...grpc.CallOption) (*milvuspb.ShowCollectionsResponse, error) ShowCollections(ctx context.Context, in *milvuspb.ShowCollectionsRequest, opts ...grpc.CallOption) (*milvuspb.ShowCollectionsResponse, error)
AlterCollection(ctx context.Context, in *milvuspb.AlterCollectionRequest, opts ...grpc.CallOption) (*commonpb.Status, error) AlterCollection(ctx context.Context, in *milvuspb.AlterCollectionRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
// * //*
// @brief This method is used to create partition // @brief This method is used to create partition
// //
// @return Status // @return Status
CreatePartition(ctx context.Context, in *milvuspb.CreatePartitionRequest, opts ...grpc.CallOption) (*commonpb.Status, error) CreatePartition(ctx context.Context, in *milvuspb.CreatePartitionRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
// * //*
// @brief This method is used to drop partition // @brief This method is used to drop partition
// //
// @return Status // @return Status
DropPartition(ctx context.Context, in *milvuspb.DropPartitionRequest, opts ...grpc.CallOption) (*commonpb.Status, error) DropPartition(ctx context.Context, in *milvuspb.DropPartitionRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
// * //*
// @brief This method is used to test partition existence. // @brief This method is used to test partition existence.
// //
// @return BoolResponse // @return BoolResponse
HasPartition(ctx context.Context, in *milvuspb.HasPartitionRequest, opts ...grpc.CallOption) (*milvuspb.BoolResponse, error) HasPartition(ctx context.Context, in *milvuspb.HasPartitionRequest, opts ...grpc.CallOption) (*milvuspb.BoolResponse, error)
// * //*
// @brief This method is used to show partition information // @brief This method is used to show partition information
// //
// @param ShowPartitionRequest, target collection name. // @param ShowPartitionRequest, target collection name.
@ -854,7 +854,7 @@ type RootCoordClient interface {
// @return StringListResponse // @return StringListResponse
ShowPartitions(ctx context.Context, in *milvuspb.ShowPartitionsRequest, opts ...grpc.CallOption) (*milvuspb.ShowPartitionsResponse, error) ShowPartitions(ctx context.Context, in *milvuspb.ShowPartitionsRequest, opts ...grpc.CallOption) (*milvuspb.ShowPartitionsResponse, error)
ShowPartitionsInternal(ctx context.Context, in *milvuspb.ShowPartitionsRequest, opts ...grpc.CallOption) (*milvuspb.ShowPartitionsResponse, error) ShowPartitionsInternal(ctx context.Context, in *milvuspb.ShowPartitionsRequest, opts ...grpc.CallOption) (*milvuspb.ShowPartitionsResponse, error)
// rpc DescribeSegment(milvus.DescribeSegmentRequest) returns (milvus.DescribeSegmentResponse) {} // rpc DescribeSegment(milvus.DescribeSegmentRequest) returns (milvus.DescribeSegmentResponse) {}
ShowSegments(ctx context.Context, in *milvuspb.ShowSegmentsRequest, opts ...grpc.CallOption) (*milvuspb.ShowSegmentsResponse, error) ShowSegments(ctx context.Context, in *milvuspb.ShowSegmentsRequest, opts ...grpc.CallOption) (*milvuspb.ShowSegmentsResponse, error)
AllocTimestamp(ctx context.Context, in *AllocTimestampRequest, opts ...grpc.CallOption) (*AllocTimestampResponse, error) AllocTimestamp(ctx context.Context, in *AllocTimestampRequest, opts ...grpc.CallOption) (*AllocTimestampResponse, error)
AllocID(ctx context.Context, in *AllocIDRequest, opts ...grpc.CallOption) (*AllocIDResponse, error) AllocID(ctx context.Context, in *AllocIDRequest, opts ...grpc.CallOption) (*AllocIDResponse, error)
@ -1327,28 +1327,28 @@ type RootCoordServer interface {
GetComponentStates(context.Context, *milvuspb.GetComponentStatesRequest) (*milvuspb.ComponentStates, error) GetComponentStates(context.Context, *milvuspb.GetComponentStatesRequest) (*milvuspb.ComponentStates, error)
GetTimeTickChannel(context.Context, *internalpb.GetTimeTickChannelRequest) (*milvuspb.StringResponse, error) GetTimeTickChannel(context.Context, *internalpb.GetTimeTickChannelRequest) (*milvuspb.StringResponse, error)
GetStatisticsChannel(context.Context, *internalpb.GetStatisticsChannelRequest) (*milvuspb.StringResponse, error) GetStatisticsChannel(context.Context, *internalpb.GetStatisticsChannelRequest) (*milvuspb.StringResponse, error)
// * //*
// @brief This method is used to create collection // @brief This method is used to create collection
// //
// @param CreateCollectionRequest, use to provide collection information to be created. // @param CreateCollectionRequest, use to provide collection information to be created.
// //
// @return Status // @return Status
CreateCollection(context.Context, *milvuspb.CreateCollectionRequest) (*commonpb.Status, error) CreateCollection(context.Context, *milvuspb.CreateCollectionRequest) (*commonpb.Status, error)
// * //*
// @brief This method is used to delete collection. // @brief This method is used to delete collection.
// //
// @param DropCollectionRequest, collection name is going to be deleted. // @param DropCollectionRequest, collection name is going to be deleted.
// //
// @return Status // @return Status
DropCollection(context.Context, *milvuspb.DropCollectionRequest) (*commonpb.Status, error) DropCollection(context.Context, *milvuspb.DropCollectionRequest) (*commonpb.Status, error)
// * //*
// @brief This method is used to test collection existence. // @brief This method is used to test collection existence.
// //
// @param HasCollectionRequest, collection name is going to be tested. // @param HasCollectionRequest, collection name is going to be tested.
// //
// @return BoolResponse // @return BoolResponse
HasCollection(context.Context, *milvuspb.HasCollectionRequest) (*milvuspb.BoolResponse, error) HasCollection(context.Context, *milvuspb.HasCollectionRequest) (*milvuspb.BoolResponse, error)
// * //*
// @brief This method is used to get collection schema. // @brief This method is used to get collection schema.
// //
// @param DescribeCollectionRequest, target collection name. // @param DescribeCollectionRequest, target collection name.
@ -1359,28 +1359,28 @@ type RootCoordServer interface {
CreateAlias(context.Context, *milvuspb.CreateAliasRequest) (*commonpb.Status, error) CreateAlias(context.Context, *milvuspb.CreateAliasRequest) (*commonpb.Status, error)
DropAlias(context.Context, *milvuspb.DropAliasRequest) (*commonpb.Status, error) DropAlias(context.Context, *milvuspb.DropAliasRequest) (*commonpb.Status, error)
AlterAlias(context.Context, *milvuspb.AlterAliasRequest) (*commonpb.Status, error) AlterAlias(context.Context, *milvuspb.AlterAliasRequest) (*commonpb.Status, error)
// * //*
// @brief This method is used to list all collections. // @brief This method is used to list all collections.
// //
// @return StringListResponse, collection name list // @return StringListResponse, collection name list
ShowCollections(context.Context, *milvuspb.ShowCollectionsRequest) (*milvuspb.ShowCollectionsResponse, error) ShowCollections(context.Context, *milvuspb.ShowCollectionsRequest) (*milvuspb.ShowCollectionsResponse, error)
AlterCollection(context.Context, *milvuspb.AlterCollectionRequest) (*commonpb.Status, error) AlterCollection(context.Context, *milvuspb.AlterCollectionRequest) (*commonpb.Status, error)
// * //*
// @brief This method is used to create partition // @brief This method is used to create partition
// //
// @return Status // @return Status
CreatePartition(context.Context, *milvuspb.CreatePartitionRequest) (*commonpb.Status, error) CreatePartition(context.Context, *milvuspb.CreatePartitionRequest) (*commonpb.Status, error)
// * //*
// @brief This method is used to drop partition // @brief This method is used to drop partition
// //
// @return Status // @return Status
DropPartition(context.Context, *milvuspb.DropPartitionRequest) (*commonpb.Status, error) DropPartition(context.Context, *milvuspb.DropPartitionRequest) (*commonpb.Status, error)
// * //*
// @brief This method is used to test partition existence. // @brief This method is used to test partition existence.
// //
// @return BoolResponse // @return BoolResponse
HasPartition(context.Context, *milvuspb.HasPartitionRequest) (*milvuspb.BoolResponse, error) HasPartition(context.Context, *milvuspb.HasPartitionRequest) (*milvuspb.BoolResponse, error)
// * //*
// @brief This method is used to show partition information // @brief This method is used to show partition information
// //
// @param ShowPartitionRequest, target collection name. // @param ShowPartitionRequest, target collection name.
@ -1388,7 +1388,7 @@ type RootCoordServer interface {
// @return StringListResponse // @return StringListResponse
ShowPartitions(context.Context, *milvuspb.ShowPartitionsRequest) (*milvuspb.ShowPartitionsResponse, error) ShowPartitions(context.Context, *milvuspb.ShowPartitionsRequest) (*milvuspb.ShowPartitionsResponse, error)
ShowPartitionsInternal(context.Context, *milvuspb.ShowPartitionsRequest) (*milvuspb.ShowPartitionsResponse, error) ShowPartitionsInternal(context.Context, *milvuspb.ShowPartitionsRequest) (*milvuspb.ShowPartitionsResponse, error)
// rpc DescribeSegment(milvus.DescribeSegmentRequest) returns (milvus.DescribeSegmentResponse) {} // rpc DescribeSegment(milvus.DescribeSegmentRequest) returns (milvus.DescribeSegmentResponse) {}
ShowSegments(context.Context, *milvuspb.ShowSegmentsRequest) (*milvuspb.ShowSegmentsResponse, error) ShowSegments(context.Context, *milvuspb.ShowSegmentsRequest) (*milvuspb.ShowSegmentsResponse, error)
AllocTimestamp(context.Context, *AllocTimestampRequest) (*AllocTimestampResponse, error) AllocTimestamp(context.Context, *AllocTimestampRequest) (*AllocTimestampResponse, error)
AllocID(context.Context, *AllocIDRequest) (*AllocIDResponse, error) AllocID(context.Context, *AllocIDRequest) (*AllocIDResponse, error)

View File

@ -880,14 +880,23 @@ func createQueryPlan(schemaPb *schemapb.CollectionSchema, exprStr string, vector
fieldID := vectorField.FieldID fieldID := vectorField.FieldID
dataType := vectorField.DataType dataType := vectorField.DataType
var vectorType planpb.VectorType
if !typeutil.IsVectorType(dataType) { if !typeutil.IsVectorType(dataType) {
return nil, fmt.Errorf("field (%s) to search is not of vector data type", vectorFieldName) return nil, fmt.Errorf("field (%s) to search is not of vector data type", vectorFieldName)
} else {
if dataType == schemapb.DataType_FloatVector {
vectorType = planpb.VectorType_FloatVector
} else if dataType == schemapb.DataType_BinaryVector {
vectorType = planpb.VectorType_BinaryVector
} else {
vectorType = planpb.VectorType_Float16Vector
}
} }
planNode := &planpb.PlanNode{ planNode := &planpb.PlanNode{
Node: &planpb.PlanNode_VectorAnns{ Node: &planpb.PlanNode_VectorAnns{
VectorAnns: &planpb.VectorANNS{ VectorAnns: &planpb.VectorANNS{
IsBinary: dataType == schemapb.DataType_BinaryVector, VectorType: vectorType,
Predicates: expr, Predicates: expr,
QueryInfo: queryInfo, QueryInfo: queryInfo,
PlaceholderTag: "$0", PlaceholderTag: "$0",

View File

@ -265,7 +265,7 @@ func (cct *createCollectionTask) PreExecute(ctx context.Context) error {
return err return err
} }
// validate vector field type parameters // validate vector field type parameters
if field.DataType == schemapb.DataType_FloatVector || field.DataType == schemapb.DataType_BinaryVector { if isVectorType(field.DataType) {
err = validateDimension(field) err = validateDimension(field)
if err != nil { if err != nil {
return err return err
@ -1426,7 +1426,7 @@ func (lct *loadCollectionTask) Execute(ctx context.Context) (err error) {
for _, index := range indexResponse.IndexInfos { for _, index := range indexResponse.IndexInfos {
fieldIndexIDs[index.FieldID] = index.IndexID fieldIndexIDs[index.FieldID] = index.IndexID
for _, field := range collSchema.Fields { for _, field := range collSchema.Fields {
if index.FieldID == field.FieldID && (field.DataType == schemapb.DataType_FloatVector || field.DataType == schemapb.DataType_BinaryVector) { if index.FieldID == field.FieldID && (field.DataType == schemapb.DataType_FloatVector || field.DataType == schemapb.DataType_BinaryVector || field.DataType == schemapb.DataType_Float16Vector) {
hasVecIndex = true hasVecIndex = true
} }
} }
@ -1654,7 +1654,7 @@ func (lpt *loadPartitionsTask) Execute(ctx context.Context) error {
for _, index := range indexResponse.IndexInfos { for _, index := range indexResponse.IndexInfos {
fieldIndexIDs[index.FieldID] = index.IndexID fieldIndexIDs[index.FieldID] = index.IndexID
for _, field := range collSchema.Fields { for _, field := range collSchema.Fields {
if index.FieldID == field.FieldID && (field.DataType == schemapb.DataType_FloatVector || field.DataType == schemapb.DataType_BinaryVector) { if index.FieldID == field.FieldID && (field.DataType == schemapb.DataType_FloatVector || field.DataType == schemapb.DataType_BinaryVector || field.DataType == schemapb.DataType_Float16Vector) {
hasVecIndex = true hasVecIndex = true
} }
} }

View File

@ -291,6 +291,7 @@ func fillDimension(field *schemapb.FieldSchema, indexParams map[string]string) e
vecDataTypes := []schemapb.DataType{ vecDataTypes := []schemapb.DataType{
schemapb.DataType_FloatVector, schemapb.DataType_FloatVector,
schemapb.DataType_BinaryVector, schemapb.DataType_BinaryVector,
schemapb.DataType_Float16Vector,
} }
if !funcutil.SliceContain(vecDataTypes, field.GetDataType()) { if !funcutil.SliceContain(vecDataTypes, field.GetDataType()) {
return nil return nil
@ -319,6 +320,7 @@ func checkTrain(field *schemapb.FieldSchema, indexParams map[string]string) erro
vecDataTypes := []schemapb.DataType{ vecDataTypes := []schemapb.DataType{
schemapb.DataType_FloatVector, schemapb.DataType_FloatVector,
schemapb.DataType_BinaryVector, schemapb.DataType_BinaryVector,
schemapb.DataType_Float16Vector,
} }
if !funcutil.SliceContain(vecDataTypes, field.GetDataType()) { if !funcutil.SliceContain(vecDataTypes, field.GetDataType()) {
return indexparamcheck.CheckIndexValid(field.GetDataType(), indexType, indexParams) return indexparamcheck.CheckIndexValid(field.GetDataType(), indexType, indexParams)

View File

@ -72,7 +72,7 @@ func translateToOutputFieldIDs(outputFields []string, schema *schemapb.Collectio
outputFieldIDs := make([]UniqueID, 0, len(outputFields)+1) outputFieldIDs := make([]UniqueID, 0, len(outputFields)+1)
if len(outputFields) == 0 { if len(outputFields) == 0 {
for _, field := range schema.Fields { for _, field := range schema.Fields {
if field.FieldID >= common.StartOfUserFieldID && field.DataType != schemapb.DataType_FloatVector && field.DataType != schemapb.DataType_BinaryVector { if field.FieldID >= common.StartOfUserFieldID && field.DataType != schemapb.DataType_FloatVector && field.DataType != schemapb.DataType_BinaryVector && field.DataType != schemapb.DataType_Float16Vector {
outputFieldIDs = append(outputFieldIDs, field.FieldID) outputFieldIDs = append(outputFieldIDs, field.FieldID)
} }
} }

View File

@ -65,6 +65,7 @@ const (
testVarCharField = "varChar" testVarCharField = "varChar"
testFloatVecField = "fvec" testFloatVecField = "fvec"
testBinaryVecField = "bvec" testBinaryVecField = "bvec"
testFloat16VecField = "f16vec"
testVecDim = 128 testVecDim = 128
testMaxVarCharLength = 100 testMaxVarCharLength = 100
) )
@ -173,7 +174,7 @@ func constructCollectionSchemaByDataType(collectionName string, fieldName2DataTy
Name: fieldName, Name: fieldName,
DataType: dataType, DataType: dataType,
} }
if dataType == schemapb.DataType_FloatVector || dataType == schemapb.DataType_BinaryVector { if dataType == schemapb.DataType_FloatVector || dataType == schemapb.DataType_BinaryVector || dataType == schemapb.DataType_Float16Vector {
fieldSchema.TypeParams = []*commonpb.KeyValuePair{ fieldSchema.TypeParams = []*commonpb.KeyValuePair{
{ {
Key: common.DimKey, Key: common.DimKey,
@ -205,7 +206,7 @@ func constructCollectionSchemaByDataType(collectionName string, fieldName2DataTy
func constructCollectionSchemaWithAllType( func constructCollectionSchemaWithAllType(
boolField, int32Field, int64Field, floatField, doubleField string, boolField, int32Field, int64Field, floatField, doubleField string,
floatVecField, binaryVecField string, floatVecField, binaryVecField, float16VecField string,
dim int, dim int,
collectionName string, collectionName string,
) *schemapb.CollectionSchema { ) *schemapb.CollectionSchema {
@ -290,6 +291,21 @@ func constructCollectionSchemaWithAllType(
IndexParams: nil, IndexParams: nil,
AutoID: false, AutoID: false,
} }
f16Vec := &schemapb.FieldSchema{
FieldID: 0,
Name: float16VecField,
IsPrimaryKey: false,
Description: "",
DataType: schemapb.DataType_Float16Vector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: common.DimKey,
Value: strconv.Itoa(dim),
},
},
IndexParams: nil,
AutoID: false,
}
if enableMultipleVectorFields { if enableMultipleVectorFields {
return &schemapb.CollectionSchema{ return &schemapb.CollectionSchema{
@ -304,6 +320,7 @@ func constructCollectionSchemaWithAllType(
d, d,
fVec, fVec,
bVec, bVec,
f16Vec,
}, },
} }
} }
@ -409,10 +426,11 @@ func constructSearchRequest(
func TestTranslateOutputFields(t *testing.T) { func TestTranslateOutputFields(t *testing.T) {
const ( const (
idFieldName = "id" idFieldName = "id"
tsFieldName = "timestamp" tsFieldName = "timestamp"
floatVectorFieldName = "float_vector" floatVectorFieldName = "float_vector"
binaryVectorFieldName = "binary_vector" binaryVectorFieldName = "binary_vector"
float16VectorFieldName = "float16_vector"
) )
var outputFields []string var outputFields []string
var userOutputFields []string var userOutputFields []string
@ -427,6 +445,7 @@ func TestTranslateOutputFields(t *testing.T) {
{Name: tsFieldName, FieldID: 1, DataType: schemapb.DataType_Int64}, {Name: tsFieldName, FieldID: 1, DataType: schemapb.DataType_Int64},
{Name: floatVectorFieldName, FieldID: 100, DataType: schemapb.DataType_FloatVector}, {Name: floatVectorFieldName, FieldID: 100, DataType: schemapb.DataType_FloatVector},
{Name: binaryVectorFieldName, FieldID: 101, DataType: schemapb.DataType_BinaryVector}, {Name: binaryVectorFieldName, FieldID: 101, DataType: schemapb.DataType_BinaryVector},
{Name: float16VectorFieldName, FieldID: 102, DataType: schemapb.DataType_Float16Vector},
}, },
} }
@ -452,23 +471,23 @@ func TestTranslateOutputFields(t *testing.T) {
outputFields, userOutputFields, err = translateOutputFields([]string{"*"}, schema, false) outputFields, userOutputFields, err = translateOutputFields([]string{"*"}, schema, false)
assert.Equal(t, nil, err) assert.Equal(t, nil, err)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, outputFields) assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, outputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, userOutputFields) assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, userOutputFields)
outputFields, userOutputFields, err = translateOutputFields([]string{" * "}, schema, false) outputFields, userOutputFields, err = translateOutputFields([]string{" * "}, schema, false)
assert.Equal(t, nil, err) assert.Equal(t, nil, err)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, outputFields) assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, outputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, userOutputFields) assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, userOutputFields)
outputFields, userOutputFields, err = translateOutputFields([]string{"*", tsFieldName}, schema, false) outputFields, userOutputFields, err = translateOutputFields([]string{"*", tsFieldName}, schema, false)
assert.Equal(t, nil, err) assert.Equal(t, nil, err)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, outputFields) assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, outputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, userOutputFields) assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, userOutputFields)
outputFields, userOutputFields, err = translateOutputFields([]string{"*", floatVectorFieldName}, schema, false) outputFields, userOutputFields, err = translateOutputFields([]string{"*", floatVectorFieldName}, schema, false)
assert.Equal(t, nil, err) assert.Equal(t, nil, err)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, outputFields) assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, outputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, userOutputFields) assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, userOutputFields)
//========================================================================= //=========================================================================
outputFields, userOutputFields, err = translateOutputFields([]string{}, schema, true) outputFields, userOutputFields, err = translateOutputFields([]string{}, schema, true)
@ -493,18 +512,18 @@ func TestTranslateOutputFields(t *testing.T) {
outputFields, userOutputFields, err = translateOutputFields([]string{"*"}, schema, true) outputFields, userOutputFields, err = translateOutputFields([]string{"*"}, schema, true)
assert.Equal(t, nil, err) assert.Equal(t, nil, err)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, outputFields) assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, outputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, userOutputFields) assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, userOutputFields)
outputFields, userOutputFields, err = translateOutputFields([]string{"*", tsFieldName}, schema, true) outputFields, userOutputFields, err = translateOutputFields([]string{"*", tsFieldName}, schema, true)
assert.Equal(t, nil, err) assert.Equal(t, nil, err)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, outputFields) assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, outputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, userOutputFields) assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, userOutputFields)
outputFields, userOutputFields, err = translateOutputFields([]string{"*", floatVectorFieldName}, schema, true) outputFields, userOutputFields, err = translateOutputFields([]string{"*", floatVectorFieldName}, schema, true)
assert.Equal(t, nil, err) assert.Equal(t, nil, err)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, outputFields) assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, outputFields)
assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName}, userOutputFields) assert.ElementsMatch(t, []string{idFieldName, tsFieldName, floatVectorFieldName, binaryVectorFieldName, float16VectorFieldName}, userOutputFields)
outputFields, userOutputFields, err = translateOutputFields([]string{"A"}, schema, true) outputFields, userOutputFields, err = translateOutputFields([]string{"A"}, schema, true)
assert.Error(t, err) assert.Error(t, err)

View File

@ -84,6 +84,12 @@ func isNumber(c uint8) bool {
return true return true
} }
func isVectorType(dataType schemapb.DataType) bool {
return dataType == schemapb.DataType_FloatVector ||
dataType == schemapb.DataType_BinaryVector ||
dataType == schemapb.DataType_Float16Vector
}
func validateMaxQueryResultWindow(offset int64, limit int64) error { func validateMaxQueryResultWindow(offset int64, limit int64) error {
if offset < 0 { if offset < 0 {
return fmt.Errorf("%s [%d] is invalid, should be gte than 0", OffsetKey, offset) return fmt.Errorf("%s [%d] is invalid, should be gte than 0", OffsetKey, offset)
@ -319,7 +325,7 @@ func validateMaxLengthPerRow(collectionName string, field *schemapb.FieldSchema)
} }
func validateVectorFieldMetricType(field *schemapb.FieldSchema) error { func validateVectorFieldMetricType(field *schemapb.FieldSchema) error {
if (field.DataType != schemapb.DataType_FloatVector) && (field.DataType != schemapb.DataType_BinaryVector) { if !isVectorType(field.DataType) {
return nil return nil
} }
for _, params := range field.IndexParams { for _, params := range field.IndexParams {
@ -432,7 +438,7 @@ func isVector(dataType schemapb.DataType) (bool, error) {
schemapb.DataType_Float, schemapb.DataType_Double: schemapb.DataType_Float, schemapb.DataType_Double:
return false, nil return false, nil
case schemapb.DataType_FloatVector, schemapb.DataType_BinaryVector: case schemapb.DataType_FloatVector, schemapb.DataType_BinaryVector, schemapb.DataType_Float16Vector:
return true, nil return true, nil
} }
@ -443,7 +449,7 @@ func validateMetricType(dataType schemapb.DataType, metricTypeStrRaw string) err
metricTypeStr := strings.ToUpper(metricTypeStrRaw) metricTypeStr := strings.ToUpper(metricTypeStrRaw)
switch metricTypeStr { switch metricTypeStr {
case metric.L2, metric.IP, metric.COSINE: case metric.L2, metric.IP, metric.COSINE:
if dataType == schemapb.DataType_FloatVector { if dataType == schemapb.DataType_FloatVector || dataType == schemapb.DataType_Float16Vector {
return nil return nil
} }
case metric.JACCARD, metric.HAMMING, metric.SUBSTRUCTURE, metric.SUPERSTRUCTURE: case metric.JACCARD, metric.HAMMING, metric.SUBSTRUCTURE, metric.SUPERSTRUCTURE:
@ -548,7 +554,7 @@ func validateMultipleVectorFields(schema *schemapb.CollectionSchema) error {
for i := range schema.Fields { for i := range schema.Fields {
name := schema.Fields[i].Name name := schema.Fields[i].Name
dType := schema.Fields[i].DataType dType := schema.Fields[i].DataType
isVec := dType == schemapb.DataType_BinaryVector || dType == schemapb.DataType_FloatVector isVec := dType == schemapb.DataType_BinaryVector || dType == schemapb.DataType_FloatVector || dType == schemapb.DataType_Float16Vector
if isVec && vecExist && !enableMultipleVectorFields { if isVec && vecExist && !enableMultipleVectorFields {
return fmt.Errorf( return fmt.Errorf(
"multiple vector fields is not supported, fields name: %s, %s", "multiple vector fields is not supported, fields name: %s, %s",

View File

@ -63,6 +63,10 @@ func (v *validateUtil) Validate(data []*schemapb.FieldData, schema *schemapb.Col
if err := v.checkFloatVectorFieldData(field, fieldSchema); err != nil { if err := v.checkFloatVectorFieldData(field, fieldSchema); err != nil {
return err return err
} }
case schemapb.DataType_Float16Vector:
if err := v.checkFloat16VectorFieldData(field, fieldSchema); err != nil {
return err
}
case schemapb.DataType_BinaryVector: case schemapb.DataType_BinaryVector:
if err := v.checkBinaryVectorFieldData(field, fieldSchema); err != nil { if err := v.checkBinaryVectorFieldData(field, fieldSchema); err != nil {
return err return err
@ -143,6 +147,26 @@ func (v *validateUtil) checkAligned(data []*schemapb.FieldData, schema *typeutil
return errNumRowsMismatch(field.GetFieldName(), n, numRows) return errNumRowsMismatch(field.GetFieldName(), n, numRows)
} }
case schemapb.DataType_Float16Vector:
f, err := schema.GetFieldFromName(field.GetFieldName())
if err != nil {
return err
}
dim, err := typeutil.GetDim(f)
if err != nil {
return err
}
n, err := funcutil.GetNumRowsOfFloat16VectorField(field.GetVectors().GetFloat16Vector(), dim)
if err != nil {
return err
}
if n != numRows {
return errNumRowsMismatch(field.GetFieldName(), n, numRows)
}
default: default:
// error won't happen here. // error won't happen here.
n, err := funcutil.GetNumRowOfFieldData(field) n, err := funcutil.GetNumRowOfFieldData(field)
@ -249,6 +273,11 @@ func (v *validateUtil) checkFloatVectorFieldData(field *schemapb.FieldData, fiel
return nil return nil
} }
func (v *validateUtil) checkFloat16VectorFieldData(field *schemapb.FieldData, fieldSchema *schemapb.FieldSchema) error {
// TODO
return nil
}
func (v *validateUtil) checkBinaryVectorFieldData(field *schemapb.FieldData, fieldSchema *schemapb.FieldSchema) error { func (v *validateUtil) checkBinaryVectorFieldData(field *schemapb.FieldData, fieldSchema *schemapb.FieldSchema) error {
// TODO // TODO
return nil return nil

View File

@ -523,6 +523,131 @@ func Test_validateUtil_checkAligned(t *testing.T) {
assert.Error(t, err) assert.Error(t, err)
}) })
//////////////////////////////////////////////////////////////////////
t.Run("float16 vector column not found", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Type: schemapb.DataType_Float16Vector,
},
}
schema := &schemapb.CollectionSchema{}
h, err := typeutil.CreateSchemaHelper(schema)
assert.NoError(t, err)
v := newValidateUtil()
err = v.checkAligned(data, h, 100)
assert.Error(t, err)
})
t.Run("float16 vector column dimension not found", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Type: schemapb.DataType_Float16Vector,
},
}
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{
Name: "test",
DataType: schemapb.DataType_Float16Vector,
},
},
}
h, err := typeutil.CreateSchemaHelper(schema)
assert.NoError(t, err)
v := newValidateUtil()
err = v.checkAligned(data, h, 100)
assert.Error(t, err)
})
t.Run("invalid num rows", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Type: schemapb.DataType_Float16Vector,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Data: &schemapb.VectorField_Float16Vector{
Float16Vector: []byte("not128"),
},
},
},
},
}
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{
Name: "test",
DataType: schemapb.DataType_Float16Vector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: common.DimKey,
Value: "128",
},
},
},
},
}
h, err := typeutil.CreateSchemaHelper(schema)
assert.NoError(t, err)
v := newValidateUtil()
err = v.checkAligned(data, h, 100)
assert.Error(t, err)
})
t.Run("num rows mismatch", func(t *testing.T) {
data := []*schemapb.FieldData{
{
FieldName: "test",
Type: schemapb.DataType_Float16Vector,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Data: &schemapb.VectorField_Float16Vector{
Float16Vector: []byte{'1', '2'},
},
},
},
},
}
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{
Name: "test",
DataType: schemapb.DataType_Float16Vector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: common.DimKey,
Value: "8",
},
},
},
},
}
h, err := typeutil.CreateSchemaHelper(schema)
assert.NoError(t, err)
v := newValidateUtil()
err = v.checkAligned(data, h, 100)
assert.Error(t, err)
})
////////////////////////////////////////////////////////////////// //////////////////////////////////////////////////////////////////
t.Run("mismatch", func(t *testing.T) { t.Run("mismatch", func(t *testing.T) {

View File

@ -114,6 +114,14 @@ var simpleBinVecField = vecFieldParam{
fieldName: "binVectorField", fieldName: "binVectorField",
} }
var simpleFloat16VecField = vecFieldParam{
id: 112,
dim: defaultDim,
metricType: defaultMetricType,
vecType: schemapb.DataType_Float16Vector,
fieldName: "float16VectorField",
}
var simpleBoolField = constFieldParam{ var simpleBoolField = constFieldParam{
id: 102, id: 102,
dataType: schemapb.DataType_Bool, dataType: schemapb.DataType_Bool,
@ -434,6 +442,16 @@ func generateBinaryVectors(numRows, dim int) []byte {
return ret return ret
} }
func generateFloat16Vectors(numRows, dim int) []byte {
total := numRows * dim * 2
ret := make([]byte, total)
_, err := rand.Read(ret)
if err != nil {
panic(err)
}
return ret
}
func GenTestScalarFieldData(dType schemapb.DataType, fieldName string, fieldID int64, numRows int) *schemapb.FieldData { func GenTestScalarFieldData(dType schemapb.DataType, fieldName string, fieldID int64, numRows int) *schemapb.FieldData {
ret := &schemapb.FieldData{ ret := &schemapb.FieldData{
Type: dType, Type: dType,
@ -589,6 +607,16 @@ func GenTestVectorFiledData(dType schemapb.DataType, fieldName string, fieldID i
}, },
}, },
} }
case schemapb.DataType_Float16Vector:
ret.FieldId = fieldID
ret.Field = &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: int64(dim),
Data: &schemapb.VectorField_Float16Vector{
Float16Vector: generateFloat16Vectors(numRows, dim),
},
},
}
default: default:
panic("data type not supported") panic("data type not supported")
} }
@ -778,6 +806,12 @@ func genInsertData(msgLength int, schema *schemapb.CollectionSchema) (*storage.I
Data: generateFloatVectors(msgLength, dim), Data: generateFloatVectors(msgLength, dim),
Dim: dim, Dim: dim,
} }
case schemapb.DataType_Float16Vector:
dim := simpleFloat16VecField.dim
insertData.Data[f.FieldID] = &storage.Float16VectorFieldData{
Data: generateFloat16Vectors(msgLength, dim),
Dim: dim,
}
case schemapb.DataType_BinaryVector: case schemapb.DataType_BinaryVector:
dim := simpleBinVecField.dim dim := simpleBinVecField.dim
insertData.Data[f.FieldID] = &storage.BinaryVectorFieldData{ insertData.Data[f.FieldID] = &storage.BinaryVectorFieldData{
@ -1042,7 +1076,7 @@ func genBruteForceDSL(schema *schemapb.CollectionSchema, topK int64, roundDecima
roundDecimalStr := strconv.FormatInt(roundDecimal, 10) roundDecimalStr := strconv.FormatInt(roundDecimal, 10)
var fieldID int64 var fieldID int64
for _, f := range schema.Fields { for _, f := range schema.Fields {
if f.DataType == schemapb.DataType_FloatVector { if f.DataType == schemapb.DataType_FloatVector || f.DataType == schemapb.DataType_Float16Vector {
vecFieldName = f.Name vecFieldName = f.Name
fieldID = f.FieldID fieldID = f.FieldID
for _, p := range f.IndexParams { for _, p := range f.IndexParams {
@ -1195,6 +1229,9 @@ func genInsertMsg(collection *Collection, partitionID, segment int64, numRows in
case schemapb.DataType_BinaryVector: case schemapb.DataType_BinaryVector:
dim := simpleBinVecField.dim // if no dim specified, use simpleFloatVecField's dim dim := simpleBinVecField.dim // if no dim specified, use simpleFloatVecField's dim
fieldsData = append(fieldsData, GenTestVectorFiledData(f.DataType, f.Name, f.FieldID, numRows, dim)) fieldsData = append(fieldsData, GenTestVectorFiledData(f.DataType, f.Name, f.FieldID, numRows, dim))
case schemapb.DataType_Float16Vector:
dim := simpleFloat16VecField.dim // if no dim specified, use simpleFloatVecField's dim
fieldsData = append(fieldsData, GenTestVectorFiledData(f.DataType, f.Name, f.FieldID, numRows, dim))
default: default:
err := errors.New("data type not supported") err := errors.New("data type not supported")
return nil, err return nil, err
@ -1429,6 +1466,20 @@ func genFieldData(fieldName string, fieldID int64, fieldType schemapb.DataType,
}, },
FieldId: fieldID, FieldId: fieldID,
} }
case schemapb.DataType_Float16Vector:
fieldData = &schemapb.FieldData{
Type: schemapb.DataType_Float16Vector,
FieldName: fieldName,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: dim,
Data: &schemapb.VectorField_Float16Vector{
Float16Vector: fieldValue.([]byte),
},
},
},
FieldId: fieldID,
}
case schemapb.DataType_JSON: case schemapb.DataType_JSON:
fieldData = &schemapb.FieldData{ fieldData = &schemapb.FieldData{
Type: schemapb.DataType_JSON, Type: schemapb.DataType_JSON,

View File

@ -148,20 +148,25 @@ type FloatVectorFieldData struct {
Data []float32 Data []float32
Dim int Dim int
} }
type Float16VectorFieldData struct {
Data []byte
Dim int
}
// RowNum implements FieldData.RowNum // RowNum implements FieldData.RowNum
func (data *BoolFieldData) RowNum() int { return len(data.Data) } func (data *BoolFieldData) RowNum() int { return len(data.Data) }
func (data *Int8FieldData) RowNum() int { return len(data.Data) } func (data *Int8FieldData) RowNum() int { return len(data.Data) }
func (data *Int16FieldData) RowNum() int { return len(data.Data) } func (data *Int16FieldData) RowNum() int { return len(data.Data) }
func (data *Int32FieldData) RowNum() int { return len(data.Data) } func (data *Int32FieldData) RowNum() int { return len(data.Data) }
func (data *Int64FieldData) RowNum() int { return len(data.Data) } func (data *Int64FieldData) RowNum() int { return len(data.Data) }
func (data *FloatFieldData) RowNum() int { return len(data.Data) } func (data *FloatFieldData) RowNum() int { return len(data.Data) }
func (data *DoubleFieldData) RowNum() int { return len(data.Data) } func (data *DoubleFieldData) RowNum() int { return len(data.Data) }
func (data *StringFieldData) RowNum() int { return len(data.Data) } func (data *StringFieldData) RowNum() int { return len(data.Data) }
func (data *BinaryVectorFieldData) RowNum() int { return len(data.Data) * 8 / data.Dim } func (data *BinaryVectorFieldData) RowNum() int { return len(data.Data) * 8 / data.Dim }
func (data *FloatVectorFieldData) RowNum() int { return len(data.Data) / data.Dim } func (data *FloatVectorFieldData) RowNum() int { return len(data.Data) / data.Dim }
func (data *ArrayFieldData) RowNum() int { return len(data.Data) } func (data *Float16VectorFieldData) RowNum() int { return len(data.Data) / 2 / data.Dim }
func (data *JSONFieldData) RowNum() int { return len(data.Data) } func (data *ArrayFieldData) RowNum() int { return len(data.Data) }
func (data *JSONFieldData) RowNum() int { return len(data.Data) }
// GetRow implements FieldData.GetRow // GetRow implements FieldData.GetRow
func (data *BoolFieldData) GetRow(i int) any { return data.Data[i] } func (data *BoolFieldData) GetRow(i int) any { return data.Data[i] }
@ -180,6 +185,9 @@ func (data *BinaryVectorFieldData) GetRow(i int) any {
func (data *FloatVectorFieldData) GetRow(i int) any { func (data *FloatVectorFieldData) GetRow(i int) any {
return data.Data[i*data.Dim : (i+1)*data.Dim] return data.Data[i*data.Dim : (i+1)*data.Dim]
} }
func (data *Float16VectorFieldData) GetRow(i int) any {
return data.Data[i*data.Dim*2 : (i+1)*data.Dim*2]
}
// why not binary.Size(data) directly? binary.Size(data) return -1 // why not binary.Size(data) directly? binary.Size(data) return -1
// binary.Size returns how many bytes Write would generate to encode the value v, which // binary.Size returns how many bytes Write would generate to encode the value v, which
@ -266,6 +274,10 @@ func (data *FloatVectorFieldData) GetMemorySize() int {
return binary.Size(data.Data) + 4 return binary.Size(data.Data) + 4
} }
func (data *Float16VectorFieldData) GetMemorySize() int {
return binary.Size(data.Data) + 4
}
// system field id: // system field id:
// 0: unique row id // 0: unique row id
// 1: timestamp // 1: timestamp
@ -438,6 +450,8 @@ func (insertCodec *InsertCodec) Serialize(partitionID UniqueID, segmentID Unique
eventWriter, err = writer.NextInsertEventWriter(singleData.(*FloatVectorFieldData).Dim) eventWriter, err = writer.NextInsertEventWriter(singleData.(*FloatVectorFieldData).Dim)
case schemapb.DataType_BinaryVector: case schemapb.DataType_BinaryVector:
eventWriter, err = writer.NextInsertEventWriter(singleData.(*BinaryVectorFieldData).Dim) eventWriter, err = writer.NextInsertEventWriter(singleData.(*BinaryVectorFieldData).Dim)
case schemapb.DataType_Float16Vector:
eventWriter, err = writer.NextInsertEventWriter(singleData.(*Float16VectorFieldData).Dim)
default: default:
return nil, fmt.Errorf("undefined data type %d", field.DataType) return nil, fmt.Errorf("undefined data type %d", field.DataType)
} }
@ -553,6 +567,14 @@ func (insertCodec *InsertCodec) Serialize(partitionID UniqueID, segmentID Unique
return nil, err return nil, err
} }
writer.AddExtra(originalSizeKey, fmt.Sprintf("%v", singleData.(*FloatVectorFieldData).GetMemorySize())) writer.AddExtra(originalSizeKey, fmt.Sprintf("%v", singleData.(*FloatVectorFieldData).GetMemorySize()))
case schemapb.DataType_Float16Vector:
err = eventWriter.AddFloat16VectorToPayload(singleData.(*Float16VectorFieldData).Data, singleData.(*Float16VectorFieldData).Dim)
if err != nil {
eventWriter.Close()
writer.Close()
return nil, err
}
writer.AddExtra(originalSizeKey, fmt.Sprintf("%v", singleData.(*Float16VectorFieldData).GetMemorySize()))
default: default:
return nil, fmt.Errorf("undefined data type %d", field.DataType) return nil, fmt.Errorf("undefined data type %d", field.DataType)
} }
@ -857,6 +879,33 @@ func (insertCodec *InsertCodec) DeserializeInto(fieldBinlogs []*Blob, rowNum int
binaryVectorFieldData.Dim = dim binaryVectorFieldData.Dim = dim
insertData.Data[fieldID] = binaryVectorFieldData insertData.Data[fieldID] = binaryVectorFieldData
case schemapb.DataType_Float16Vector:
var singleData []byte
singleData, dim, err = eventReader.GetFloat16VectorFromPayload()
if err != nil {
eventReader.Close()
binlogReader.Close()
return InvalidUniqueID, InvalidUniqueID, InvalidUniqueID, err
}
if insertData.Data[fieldID] == nil {
insertData.Data[fieldID] = &Float16VectorFieldData{
Data: make([]byte, 0, rowNum*dim),
}
}
float16VectorFieldData := insertData.Data[fieldID].(*Float16VectorFieldData)
float16VectorFieldData.Data = append(float16VectorFieldData.Data, singleData...)
length, err := eventReader.GetPayloadLengthFromReader()
if err != nil {
eventReader.Close()
binlogReader.Close()
return InvalidUniqueID, InvalidUniqueID, InvalidUniqueID, err
}
totalLength += length
float16VectorFieldData.Dim = dim
insertData.Data[fieldID] = float16VectorFieldData
case schemapb.DataType_FloatVector: case schemapb.DataType_FloatVector:
var singleData []float32 var singleData []float32
singleData, dim, err = eventReader.GetFloatVectorFromPayload() singleData, dim, err = eventReader.GetFloatVectorFromPayload()

View File

@ -30,23 +30,24 @@ import (
) )
const ( const (
CollectionID = 1 CollectionID = 1
PartitionID = 1 PartitionID = 1
SegmentID = 1 SegmentID = 1
RowIDField = 0 RowIDField = 0
TimestampField = 1 TimestampField = 1
BoolField = 100 BoolField = 100
Int8Field = 101 Int8Field = 101
Int16Field = 102 Int16Field = 102
Int32Field = 103 Int32Field = 103
Int64Field = 104 Int64Field = 104
FloatField = 105 FloatField = 105
DoubleField = 106 DoubleField = 106
StringField = 107 StringField = 107
BinaryVectorField = 108 BinaryVectorField = 108
FloatVectorField = 109 FloatVectorField = 109
ArrayField = 110 ArrayField = 110
JSONField = 111 JSONField = 111
Float16VectorField = 112
) )
func TestInsertCodec(t *testing.T) { func TestInsertCodec(t *testing.T) {
@ -157,6 +158,13 @@ func TestInsertCodec(t *testing.T) {
Description: "float_vector", Description: "float_vector",
DataType: schemapb.DataType_FloatVector, DataType: schemapb.DataType_FloatVector,
}, },
{
FieldID: Float16VectorField,
Name: "field_float16_vector",
IsPrimaryKey: false,
Description: "float16_vector",
DataType: schemapb.DataType_Float16Vector,
},
}, },
}, },
} }
@ -222,6 +230,10 @@ func TestInsertCodec(t *testing.T) {
[]byte(`{"key":"world"}`), []byte(`{"key":"world"}`),
}, },
}, },
Float16VectorField: &Float16VectorFieldData{
Data: []byte{0, 255, 0, 255, 0, 255, 0, 255},
Dim: 4,
},
}, },
} }
@ -265,6 +277,10 @@ func TestInsertCodec(t *testing.T) {
Data: []float32{0, 1, 2, 3, 0, 1, 2, 3}, Data: []float32{0, 1, 2, 3, 0, 1, 2, 3},
Dim: 4, Dim: 4,
}, },
Float16VectorField: &Float16VectorFieldData{
Data: []byte{0, 255, 0, 255, 0, 255, 0, 255},
Dim: 4,
},
ArrayField: &ArrayFieldData{ ArrayField: &ArrayFieldData{
ElementType: schemapb.DataType_Int32, ElementType: schemapb.DataType_Int32,
Data: []*schemapb.ScalarField{ Data: []*schemapb.ScalarField{
@ -291,20 +307,21 @@ func TestInsertCodec(t *testing.T) {
insertDataEmpty := &InsertData{ insertDataEmpty := &InsertData{
Data: map[int64]FieldData{ Data: map[int64]FieldData{
RowIDField: &Int64FieldData{[]int64{}}, RowIDField: &Int64FieldData{[]int64{}},
TimestampField: &Int64FieldData{[]int64{}}, TimestampField: &Int64FieldData{[]int64{}},
BoolField: &BoolFieldData{[]bool{}}, BoolField: &BoolFieldData{[]bool{}},
Int8Field: &Int8FieldData{[]int8{}}, Int8Field: &Int8FieldData{[]int8{}},
Int16Field: &Int16FieldData{[]int16{}}, Int16Field: &Int16FieldData{[]int16{}},
Int32Field: &Int32FieldData{[]int32{}}, Int32Field: &Int32FieldData{[]int32{}},
Int64Field: &Int64FieldData{[]int64{}}, Int64Field: &Int64FieldData{[]int64{}},
FloatField: &FloatFieldData{[]float32{}}, FloatField: &FloatFieldData{[]float32{}},
DoubleField: &DoubleFieldData{[]float64{}}, DoubleField: &DoubleFieldData{[]float64{}},
StringField: &StringFieldData{[]string{}}, StringField: &StringFieldData{[]string{}},
BinaryVectorField: &BinaryVectorFieldData{[]byte{}, 8}, BinaryVectorField: &BinaryVectorFieldData{[]byte{}, 8},
FloatVectorField: &FloatVectorFieldData{[]float32{}, 4}, FloatVectorField: &FloatVectorFieldData{[]float32{}, 4},
ArrayField: &ArrayFieldData{schemapb.DataType_Int32, []*schemapb.ScalarField{}}, Float16VectorField: &Float16VectorFieldData{[]byte{}, 4},
JSONField: &JSONFieldData{[][]byte{}}, ArrayField: &ArrayFieldData{schemapb.DataType_Int32, []*schemapb.ScalarField{}},
JSONField: &JSONFieldData{[][]byte{}},
}, },
} }
b, err := insertCodec.Serialize(PartitionID, SegmentID, insertDataEmpty) b, err := insertCodec.Serialize(PartitionID, SegmentID, insertDataEmpty)
@ -345,6 +362,7 @@ func TestInsertCodec(t *testing.T) {
assert.Equal(t, []string{"1", "2", "3", "4"}, resultData.Data[StringField].(*StringFieldData).Data) assert.Equal(t, []string{"1", "2", "3", "4"}, resultData.Data[StringField].(*StringFieldData).Data)
assert.Equal(t, []byte{0, 255, 0, 255}, resultData.Data[BinaryVectorField].(*BinaryVectorFieldData).Data) assert.Equal(t, []byte{0, 255, 0, 255}, resultData.Data[BinaryVectorField].(*BinaryVectorFieldData).Data)
assert.Equal(t, []float32{0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 4, 5, 6, 7}, resultData.Data[FloatVectorField].(*FloatVectorFieldData).Data) assert.Equal(t, []float32{0, 1, 2, 3, 0, 1, 2, 3, 4, 5, 6, 7, 4, 5, 6, 7}, resultData.Data[FloatVectorField].(*FloatVectorFieldData).Data)
assert.Equal(t, []byte{0, 255, 0, 255, 0, 255, 0, 255, 0, 255, 0, 255, 0, 255, 0, 255}, resultData.Data[Float16VectorField].(*Float16VectorFieldData).Data)
int32ArrayList := [][]int32{{1, 2, 3}, {4, 5, 6}, {3, 2, 1}, {6, 5, 4}} int32ArrayList := [][]int32{{1, 2, 3}, {4, 5, 6}, {3, 2, 1}, {6, 5, 4}}
resultArrayList := [][]int32{} resultArrayList := [][]int32{}

View File

@ -94,6 +94,13 @@ func (ds *DataSorter) Swap(i, j int) {
for idx := 0; idx < dim; idx++ { for idx := 0; idx < dim; idx++ {
data[i*dim+idx], data[j*dim+idx] = data[j*dim+idx], data[i*dim+idx] data[i*dim+idx], data[j*dim+idx] = data[j*dim+idx], data[i*dim+idx]
} }
case schemapb.DataType_Float16Vector:
data := singleData.(*Float16VectorFieldData).Data
dim := singleData.(*Float16VectorFieldData).Dim
steps := dim * 2
for idx := 0; idx < steps; idx++ {
data[i*steps+idx], data[j*steps+idx] = data[j*steps+idx], data[i*steps+idx]
}
case schemapb.DataType_Array: case schemapb.DataType_Array:
data := singleData.(*ArrayFieldData).Data data := singleData.(*ArrayFieldData).Data
data[i], data[j] = data[j], data[i] data[i], data[j] = data[j], data[i]

View File

@ -120,6 +120,13 @@ func TestDataSorter(t *testing.T) {
Description: "description_11", Description: "description_11",
DataType: schemapb.DataType_FloatVector, DataType: schemapb.DataType_FloatVector,
}, },
{
FieldID: 110,
Name: "field_float16_vector",
IsPrimaryKey: false,
Description: "description_12",
DataType: schemapb.DataType_Float16Vector,
},
}, },
}, },
} }
@ -165,6 +172,10 @@ func TestDataSorter(t *testing.T) {
Data: []float32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23}, Data: []float32{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23},
Dim: 8, Dim: 8,
}, },
110: &Float16VectorFieldData{
Data: []byte{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23},
Dim: 4,
},
}, },
} }
@ -226,6 +237,7 @@ func TestDataSorter(t *testing.T) {
assert.Equal(t, []string{"5", "3", "4"}, dataSorter.InsertData.Data[107].(*StringFieldData).Data) assert.Equal(t, []string{"5", "3", "4"}, dataSorter.InsertData.Data[107].(*StringFieldData).Data)
assert.Equal(t, []byte{128, 0, 255}, dataSorter.InsertData.Data[108].(*BinaryVectorFieldData).Data) assert.Equal(t, []byte{128, 0, 255}, dataSorter.InsertData.Data[108].(*BinaryVectorFieldData).Data)
assert.Equal(t, []float32{16, 17, 18, 19, 20, 21, 22, 23, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, dataSorter.InsertData.Data[109].(*FloatVectorFieldData).Data) assert.Equal(t, []float32{16, 17, 18, 19, 20, 21, 22, 23, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, dataSorter.InsertData.Data[109].(*FloatVectorFieldData).Data)
assert.Equal(t, []byte{16, 17, 18, 19, 20, 21, 22, 23, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}, dataSorter.InsertData.Data[110].(*Float16VectorFieldData).Data)
} }
func TestDataSorter_Len(t *testing.T) { func TestDataSorter_Len(t *testing.T) {

View File

@ -36,6 +36,7 @@ type PayloadWriterInterface interface {
AddOneJSONToPayload(msg []byte) error AddOneJSONToPayload(msg []byte) error
AddBinaryVectorToPayload(binVec []byte, dim int) error AddBinaryVectorToPayload(binVec []byte, dim int) error
AddFloatVectorToPayload(binVec []float32, dim int) error AddFloatVectorToPayload(binVec []float32, dim int) error
AddFloat16VectorToPayload(binVec []byte, dim int) error
FinishPayloadWriter() error FinishPayloadWriter() error
GetPayloadBufferFromWriter() ([]byte, error) GetPayloadBufferFromWriter() ([]byte, error)
GetPayloadLengthFromWriter() (int, error) GetPayloadLengthFromWriter() (int, error)
@ -58,6 +59,7 @@ type PayloadReaderInterface interface {
GetArrayFromPayload() ([]*schemapb.ScalarField, error) GetArrayFromPayload() ([]*schemapb.ScalarField, error)
GetJSONFromPayload() ([][]byte, error) GetJSONFromPayload() ([][]byte, error)
GetBinaryVectorFromPayload() ([]byte, int, error) GetBinaryVectorFromPayload() ([]byte, int, error)
GetFloat16VectorFromPayload() ([]byte, int, error)
GetFloatVectorFromPayload() ([]float32, int, error) GetFloatVectorFromPayload() ([]float32, int, error)
GetPayloadLengthFromReader() (int, error) GetPayloadLengthFromReader() (int, error)
ReleasePayloadReader() error ReleasePayloadReader() error

View File

@ -67,6 +67,8 @@ func (r *PayloadReader) GetDataFromPayload() (interface{}, int, error) {
return r.GetBinaryVectorFromPayload() return r.GetBinaryVectorFromPayload()
case schemapb.DataType_FloatVector: case schemapb.DataType_FloatVector:
return r.GetFloatVectorFromPayload() return r.GetFloatVectorFromPayload()
case schemapb.DataType_Float16Vector:
return r.GetFloat16VectorFromPayload()
case schemapb.DataType_String, schemapb.DataType_VarChar: case schemapb.DataType_String, schemapb.DataType_VarChar:
val, err := r.GetStringFromPayload() val, err := r.GetStringFromPayload()
return val, 0, err return val, 0, err
@ -315,6 +317,29 @@ func (r *PayloadReader) GetBinaryVectorFromPayload() ([]byte, int, error) {
return ret, dim * 8, nil return ret, dim * 8, nil
} }
// GetFloat16VectorFromPayload returns vector, dimension, error
func (r *PayloadReader) GetFloat16VectorFromPayload() ([]byte, int, error) {
if r.colType != schemapb.DataType_Float16Vector {
return nil, -1, fmt.Errorf("failed to get float vector from datatype %v", r.colType.String())
}
dim := r.reader.RowGroup(0).Column(0).Descriptor().TypeLength() / 2
values := make([]parquet.FixedLenByteArray, r.numRows)
valuesRead, err := ReadDataFromAllRowGroups[parquet.FixedLenByteArray, *file.FixedLenByteArrayColumnChunkReader](r.reader, values, 0, r.numRows)
if err != nil {
return nil, -1, err
}
if valuesRead != r.numRows {
return nil, -1, fmt.Errorf("expect %d rows, but got valuesRead = %d", r.numRows, valuesRead)
}
ret := make([]byte, int64(dim*2)*r.numRows)
for i := 0; i < int(r.numRows); i++ {
copy(ret[i*dim*2:(i+1)*dim*2], values[i])
}
return ret, dim, nil
}
// GetFloatVectorFromPayload returns vector, dimension, error // GetFloatVectorFromPayload returns vector, dimension, error
func (r *PayloadReader) GetFloatVectorFromPayload() ([]float32, int, error) { func (r *PayloadReader) GetFloatVectorFromPayload() ([]float32, int, error) {
if r.colType != schemapb.DataType_FloatVector { if r.colType != schemapb.DataType_FloatVector {

View File

@ -536,6 +536,39 @@ func TestPayload_ReaderAndWriter(t *testing.T) {
defer r.ReleasePayloadReader() defer r.ReleasePayloadReader()
}) })
t.Run("TestFloat16Vector", func(t *testing.T) {
w, err := NewPayloadWriter(schemapb.DataType_Float16Vector, 1)
require.Nil(t, err)
require.NotNil(t, w)
err = w.AddFloat16VectorToPayload([]byte{1, 2}, 1)
assert.NoError(t, err)
err = w.AddDataToPayload([]byte{3, 4}, 1)
assert.NoError(t, err)
err = w.FinishPayloadWriter()
assert.NoError(t, err)
length, err := w.GetPayloadLengthFromWriter()
assert.NoError(t, err)
assert.Equal(t, 2, length)
defer w.ReleasePayloadWriter()
buffer, err := w.GetPayloadBufferFromWriter()
assert.NoError(t, err)
r, err := NewPayloadReader(schemapb.DataType_Float16Vector, buffer)
require.Nil(t, err)
length, err = r.GetPayloadLengthFromReader()
assert.NoError(t, err)
assert.Equal(t, length, 2)
float16Vecs, dim, err := r.GetFloat16VectorFromPayload()
assert.NoError(t, err)
assert.Equal(t, 1, dim)
assert.Equal(t, 4, len(float16Vecs))
assert.ElementsMatch(t, []byte{1, 2, 3, 4}, float16Vecs)
})
// t.Run("TestAddDataToPayload", func(t *testing.T) { // t.Run("TestAddDataToPayload", func(t *testing.T) {
// w, err := NewPayloadWriter(schemapb.DataType_Bool) // w, err := NewPayloadWriter(schemapb.DataType_Bool)
// w.colType = 999 // w.colType = 999

View File

@ -151,6 +151,12 @@ func (w *NativePayloadWriter) AddDataToPayload(data interface{}, dim ...int) err
return errors.New("incorrect data type") return errors.New("incorrect data type")
} }
return w.AddFloatVectorToPayload(val, dim[0]) return w.AddFloatVectorToPayload(val, dim[0])
case schemapb.DataType_Float16Vector:
val, ok := data.([]byte)
if !ok {
return errors.New("incorrect data type")
}
return w.AddFloat16VectorToPayload(val, dim[0])
default: default:
return errors.New("incorrect datatype") return errors.New("incorrect datatype")
} }
@ -412,6 +418,31 @@ func (w *NativePayloadWriter) AddFloatVectorToPayload(data []float32, dim int) e
return nil return nil
} }
func (w *NativePayloadWriter) AddFloat16VectorToPayload(data []byte, dim int) error {
if w.finished {
return errors.New("can't append data to finished writer")
}
if len(data) == 0 {
return errors.New("can't add empty msgs into payload")
}
builder, ok := w.builder.(*array.FixedSizeBinaryBuilder)
if !ok {
return errors.New("failed to cast ArrayBuilder")
}
byteLength := dim * 2
length := len(data) / byteLength
builder.Reserve(length)
for i := 0; i < length; i++ {
builder.Append(data[i*byteLength : (i+1)*byteLength])
}
return nil
}
func (w *NativePayloadWriter) FinishPayloadWriter() error { func (w *NativePayloadWriter) FinishPayloadWriter() error {
if w.finished { if w.finished {
return errors.New("can't reuse a finished writer") return errors.New("can't reuse a finished writer")
@ -503,6 +534,10 @@ func milvusDataTypeToArrowType(dataType schemapb.DataType, dim int) arrow.DataTy
return &arrow.FixedSizeBinaryType{ return &arrow.FixedSizeBinaryType{
ByteWidth: dim / 8, ByteWidth: dim / 8,
} }
case schemapb.DataType_Float16Vector:
return &arrow.FixedSizeBinaryType{
ByteWidth: dim * 2,
}
default: default:
panic("unsupported data type") panic("unsupported data type")
} }

View File

@ -224,6 +224,16 @@ func readBinaryVectors(blobReaders []io.Reader, dim int) []byte {
return ret return ret
} }
func readFloat16Vectors(blobReaders []io.Reader, dim int) []byte {
ret := make([]byte, 0)
for _, r := range blobReaders {
var v = make([]byte, dim*2)
ReadBinary(r, &v, schemapb.DataType_Float16Vector)
ret = append(ret, v...)
}
return ret
}
func readBoolArray(blobReaders []io.Reader) []bool { func readBoolArray(blobReaders []io.Reader) []bool {
ret := make([]bool, 0) ret := make([]bool, 0)
for _, r := range blobReaders { for _, r := range blobReaders {
@ -321,6 +331,19 @@ func RowBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *schemap
Dim: dim, Dim: dim,
} }
case schemapb.DataType_Float16Vector:
dim, err := GetDimFromParams(field.TypeParams)
if err != nil {
log.Error("failed to get dim", zap.Error(err))
return nil, err
}
vecs := readFloat16Vectors(blobReaders, dim)
idata.Data[field.FieldID] = &Float16VectorFieldData{
Data: vecs,
Dim: dim,
}
case schemapb.DataType_BinaryVector: case schemapb.DataType_BinaryVector:
var dim int var dim int
dim, err := GetDimFromParams(field.TypeParams) dim, err := GetDimFromParams(field.TypeParams)
@ -435,6 +458,23 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
idata.Data[field.FieldID] = fieldData idata.Data[field.FieldID] = fieldData
case schemapb.DataType_Float16Vector:
dim, err := GetDimFromParams(field.TypeParams)
if err != nil {
log.Error("failed to get dim", zap.Error(err))
return nil, err
}
srcData := srcFields[field.FieldID].GetVectors().GetFloat16Vector()
fieldData := &Float16VectorFieldData{
Data: make([]byte, 0, len(srcData)),
Dim: dim,
}
fieldData.Data = append(fieldData.Data, srcData...)
idata.Data[field.FieldID] = fieldData
case schemapb.DataType_Bool: case schemapb.DataType_Bool:
srcData := srcFields[field.FieldID].GetScalars().GetBoolData().GetData() srcData := srcFields[field.FieldID].GetScalars().GetBoolData().GetData()
@ -698,6 +738,18 @@ func mergeFloatVectorField(data *InsertData, fid FieldID, field *FloatVectorFiel
fieldData.Data = append(fieldData.Data, field.Data...) fieldData.Data = append(fieldData.Data, field.Data...)
} }
func mergeFloat16VectorField(data *InsertData, fid FieldID, field *Float16VectorFieldData) {
if _, ok := data.Data[fid]; !ok {
fieldData := &Float16VectorFieldData{
Data: nil,
Dim: field.Dim,
}
data.Data[fid] = fieldData
}
fieldData := data.Data[fid].(*Float16VectorFieldData)
fieldData.Data = append(fieldData.Data, field.Data...)
}
// MergeFieldData merge field into data. // MergeFieldData merge field into data.
func MergeFieldData(data *InsertData, fid FieldID, field FieldData) { func MergeFieldData(data *InsertData, fid FieldID, field FieldData) {
if field == nil { if field == nil {
@ -728,6 +780,8 @@ func MergeFieldData(data *InsertData, fid FieldID, field FieldData) {
mergeBinaryVectorField(data, fid, field) mergeBinaryVectorField(data, fid, field)
case *FloatVectorFieldData: case *FloatVectorFieldData:
mergeFloatVectorField(data, fid, field) mergeFloatVectorField(data, fid, field)
case *Float16VectorFieldData:
mergeFloat16VectorField(data, fid, field)
} }
} }

View File

@ -169,6 +169,10 @@ func TestTransferColumnBasedInsertDataToRowBased(t *testing.T) {
Dim: 1, Dim: 1,
Data: []float32{0, 0, 0}, Data: []float32{0, 0, 0},
} }
f11 := &Float16VectorFieldData{
Dim: 1,
Data: []byte{1, 1, 2, 2, 3, 3},
}
data.Data[101] = f1 data.Data[101] = f1
data.Data[102] = f2 data.Data[102] = f2
@ -180,6 +184,7 @@ func TestTransferColumnBasedInsertDataToRowBased(t *testing.T) {
// data.Data[108] = f8 // data.Data[108] = f8
data.Data[109] = f9 data.Data[109] = f9
data.Data[110] = f10 data.Data[110] = f10
data.Data[111] = f11
utss, rowIds, rows, err := TransferColumnBasedInsertDataToRowBased(data) utss, rowIds, rows, err := TransferColumnBasedInsertDataToRowBased(data)
assert.NoError(t, err) assert.NoError(t, err)
@ -202,6 +207,7 @@ func TestTransferColumnBasedInsertDataToRowBased(t *testing.T) {
// b + 1, // "1" // b + 1, // "1"
1, // 1 1, // 1
0, 0, 0, 0, // 0 0, 0, 0, 0, // 0
1, 1,
}, },
rows[0].Value) rows[0].Value)
assert.ElementsMatch(t, assert.ElementsMatch(t,
@ -216,6 +222,7 @@ func TestTransferColumnBasedInsertDataToRowBased(t *testing.T) {
// b + 2, // "2" // b + 2, // "2"
2, // 2 2, // 2
0, 0, 0, 0, // 0 0, 0, 0, 0, // 0
2, 2,
}, },
rows[1].Value) rows[1].Value)
assert.ElementsMatch(t, assert.ElementsMatch(t,
@ -230,6 +237,7 @@ func TestTransferColumnBasedInsertDataToRowBased(t *testing.T) {
// b + 3, // "3" // b + 3, // "3"
3, // 3 3, // 3
0, 0, 0, 0, // 0 0, 0, 0, 0, // 0
3, 3,
}, },
rows[2].Value) rows[2].Value)
} }
@ -313,7 +321,7 @@ func TestReadBinary(t *testing.T) {
} }
} }
func genAllFieldsSchema(fVecDim, bVecDim int) (schema *schemapb.CollectionSchema, pkFieldID UniqueID, fieldIDs []UniqueID) { func genAllFieldsSchema(fVecDim, bVecDim, f16VecDim int) (schema *schemapb.CollectionSchema, pkFieldID UniqueID, fieldIDs []UniqueID) {
schema = &schemapb.CollectionSchema{ schema = &schemapb.CollectionSchema{
Name: "all_fields_schema", Name: "all_fields_schema",
Description: "all_fields_schema", Description: "all_fields_schema",
@ -359,6 +367,15 @@ func genAllFieldsSchema(fVecDim, bVecDim int) (schema *schemapb.CollectionSchema
}, },
}, },
}, },
{
DataType: schemapb.DataType_Float16Vector,
TypeParams: []*commonpb.KeyValuePair{
{
Key: common.DimKey,
Value: strconv.Itoa(f16VecDim),
},
},
},
{ {
DataType: schemapb.DataType_Array, DataType: schemapb.DataType_Array,
}, },
@ -412,6 +429,16 @@ func generateBinaryVectors(numRows, dim int) []byte {
return ret return ret
} }
func generateFloat16Vectors(numRows, dim int) []byte {
total := (numRows * dim) * 2
ret := make([]byte, total)
_, err := rand.Read(ret)
if err != nil {
panic(err)
}
return ret
}
func generateBoolArray(numRows int) []bool { func generateBoolArray(numRows int) []bool {
ret := make([]bool, 0, numRows) ret := make([]bool, 0, numRows)
for i := 0; i < numRows; i++ { for i := 0; i < numRows; i++ {
@ -474,8 +501,8 @@ func generateInt32ArrayList(numRows int) []*schemapb.ScalarField {
return ret return ret
} }
func genRowWithAllFields(fVecDim, bVecDim int) (blob *commonpb.Blob, pk int64, row []interface{}) { func genRowWithAllFields(fVecDim, bVecDim, f16VecDim int) (blob *commonpb.Blob, pk int64, row []interface{}) {
schema, _, _ := genAllFieldsSchema(fVecDim, bVecDim) schema, _, _ := genAllFieldsSchema(fVecDim, bVecDim, f16VecDim)
ret := &commonpb.Blob{ ret := &commonpb.Blob{
Value: nil, Value: nil,
} }
@ -493,6 +520,11 @@ func genRowWithAllFields(fVecDim, bVecDim int) (blob *commonpb.Blob, pk int64, r
_ = binary.Write(&buffer, common.Endian, bVec) _ = binary.Write(&buffer, common.Endian, bVec)
ret.Value = append(ret.Value, buffer.Bytes()...) ret.Value = append(ret.Value, buffer.Bytes()...)
row = append(row, bVec) row = append(row, bVec)
case schemapb.DataType_Float16Vector:
f16Vec := generateFloat16Vectors(1, f16VecDim)
_ = binary.Write(&buffer, common.Endian, f16Vec)
ret.Value = append(ret.Value, buffer.Bytes()...)
row = append(row, f16Vec)
case schemapb.DataType_Bool: case schemapb.DataType_Bool:
data := rand.Int()%2 == 0 data := rand.Int()%2 == 0
_ = binary.Write(&buffer, common.Endian, data) _ = binary.Write(&buffer, common.Endian, data)
@ -550,7 +582,7 @@ func genRowWithAllFields(fVecDim, bVecDim int) (blob *commonpb.Blob, pk int64, r
return ret, pk, row return ret, pk, row
} }
func genRowBasedInsertMsg(numRows, fVecDim, bVecDim int) (msg *msgstream.InsertMsg, pks []int64, columns [][]interface{}) { func genRowBasedInsertMsg(numRows, fVecDim, bVecDim, f16VecDim int) (msg *msgstream.InsertMsg, pks []int64, columns [][]interface{}) {
msg = &msgstream.InsertMsg{ msg = &msgstream.InsertMsg{
BaseMsg: msgstream.BaseMsg{ BaseMsg: msgstream.BaseMsg{
Ctx: nil, Ctx: nil,
@ -573,7 +605,7 @@ func genRowBasedInsertMsg(numRows, fVecDim, bVecDim int) (msg *msgstream.InsertM
pks = make([]int64, 0) pks = make([]int64, 0)
raws := make([][]interface{}, 0) raws := make([][]interface{}, 0)
for i := 0; i < numRows; i++ { for i := 0; i < numRows; i++ {
row, pk, raw := genRowWithAllFields(fVecDim, bVecDim) row, pk, raw := genRowWithAllFields(fVecDim, bVecDim, f16VecDim)
msg.InsertRequest.RowData = append(msg.InsertRequest.RowData, row) msg.InsertRequest.RowData = append(msg.InsertRequest.RowData, row)
pks = append(pks, pk) pks = append(pks, pk)
raws = append(raws, raw) raws = append(raws, raw)
@ -588,7 +620,7 @@ func genRowBasedInsertMsg(numRows, fVecDim, bVecDim int) (msg *msgstream.InsertM
return msg, pks, columns return msg, pks, columns
} }
func genColumnBasedInsertMsg(schema *schemapb.CollectionSchema, numRows, fVecDim, bVecDim int) (msg *msgstream.InsertMsg, pks []int64, columns [][]interface{}) { func genColumnBasedInsertMsg(schema *schemapb.CollectionSchema, numRows, fVecDim, bVecDim, f16VecDim int) (msg *msgstream.InsertMsg, pks []int64, columns [][]interface{}) {
msg = &msgstream.InsertMsg{ msg = &msgstream.InsertMsg{
BaseMsg: msgstream.BaseMsg{ BaseMsg: msgstream.BaseMsg{
Ctx: nil, Ctx: nil,
@ -795,6 +827,25 @@ func genColumnBasedInsertMsg(schema *schemapb.CollectionSchema, numRows, fVecDim
for nrows := 0; nrows < numRows; nrows++ { for nrows := 0; nrows < numRows; nrows++ {
columns[idx] = append(columns[idx], data[nrows*bVecDim/8:(nrows+1)*bVecDim/8]) columns[idx] = append(columns[idx], data[nrows*bVecDim/8:(nrows+1)*bVecDim/8])
} }
case schemapb.DataType_Float16Vector:
data := generateFloat16Vectors(numRows, f16VecDim)
f := &schemapb.FieldData{
Type: schemapb.DataType_Float16Vector,
FieldName: field.Name,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: int64(f16VecDim),
Data: &schemapb.VectorField_Float16Vector{
Float16Vector: data,
},
},
},
FieldId: field.FieldID,
}
msg.FieldsData = append(msg.FieldsData, f)
for nrows := 0; nrows < numRows; nrows++ {
columns[idx] = append(columns[idx], data[nrows*f16VecDim*2:(nrows+1)*f16VecDim*2])
}
case schemapb.DataType_Array: case schemapb.DataType_Array:
data := generateInt32ArrayList(numRows) data := generateInt32ArrayList(numRows)
@ -845,10 +896,10 @@ func genColumnBasedInsertMsg(schema *schemapb.CollectionSchema, numRows, fVecDim
} }
func TestRowBasedInsertMsgToInsertData(t *testing.T) { func TestRowBasedInsertMsgToInsertData(t *testing.T) {
numRows, fVecDim, bVecDim := 10, 8, 8 numRows, fVecDim, bVecDim, f16VecDim := 10, 8, 8, 8
schema, _, fieldIDs := genAllFieldsSchema(fVecDim, bVecDim) schema, _, fieldIDs := genAllFieldsSchema(fVecDim, bVecDim, f16VecDim)
fieldIDs = fieldIDs[:len(fieldIDs)-2] fieldIDs = fieldIDs[:len(fieldIDs)-2]
msg, _, columns := genRowBasedInsertMsg(numRows, fVecDim, bVecDim) msg, _, columns := genRowBasedInsertMsg(numRows, fVecDim, bVecDim, f16VecDim)
idata, err := RowBasedInsertMsgToInsertData(msg, schema) idata, err := RowBasedInsertMsgToInsertData(msg, schema)
assert.NoError(t, err) assert.NoError(t, err)
@ -864,9 +915,9 @@ func TestRowBasedInsertMsgToInsertData(t *testing.T) {
} }
func TestColumnBasedInsertMsgToInsertData(t *testing.T) { func TestColumnBasedInsertMsgToInsertData(t *testing.T) {
numRows, fVecDim, bVecDim := 2, 2, 8 numRows, fVecDim, bVecDim, f16VecDim := 2, 2, 8, 2
schema, _, fieldIDs := genAllFieldsSchema(fVecDim, bVecDim) schema, _, fieldIDs := genAllFieldsSchema(fVecDim, bVecDim, f16VecDim)
msg, _, columns := genColumnBasedInsertMsg(schema, numRows, fVecDim, bVecDim) msg, _, columns := genColumnBasedInsertMsg(schema, numRows, fVecDim, bVecDim, f16VecDim)
idata, err := ColumnBasedInsertMsgToInsertData(msg, schema) idata, err := ColumnBasedInsertMsgToInsertData(msg, schema)
assert.NoError(t, err) assert.NoError(t, err)
@ -882,10 +933,10 @@ func TestColumnBasedInsertMsgToInsertData(t *testing.T) {
} }
func TestInsertMsgToInsertData(t *testing.T) { func TestInsertMsgToInsertData(t *testing.T) {
numRows, fVecDim, bVecDim := 10, 8, 8 numRows, fVecDim, bVecDim, f16VecDim := 10, 8, 8, 8
schema, _, fieldIDs := genAllFieldsSchema(fVecDim, bVecDim) schema, _, fieldIDs := genAllFieldsSchema(fVecDim, bVecDim, f16VecDim)
fieldIDs = fieldIDs[:len(fieldIDs)-2] fieldIDs = fieldIDs[:len(fieldIDs)-2]
msg, _, columns := genRowBasedInsertMsg(numRows, fVecDim, bVecDim) msg, _, columns := genRowBasedInsertMsg(numRows, fVecDim, bVecDim, f16VecDim)
idata, err := InsertMsgToInsertData(msg, schema) idata, err := InsertMsgToInsertData(msg, schema)
assert.NoError(t, err) assert.NoError(t, err)
@ -901,9 +952,9 @@ func TestInsertMsgToInsertData(t *testing.T) {
} }
func TestInsertMsgToInsertData2(t *testing.T) { func TestInsertMsgToInsertData2(t *testing.T) {
numRows, fVecDim, bVecDim := 2, 2, 8 numRows, fVecDim, bVecDim, f16VecDim := 2, 2, 8, 2
schema, _, fieldIDs := genAllFieldsSchema(fVecDim, bVecDim) schema, _, fieldIDs := genAllFieldsSchema(fVecDim, bVecDim, f16VecDim)
msg, _, columns := genColumnBasedInsertMsg(schema, numRows, fVecDim, bVecDim) msg, _, columns := genColumnBasedInsertMsg(schema, numRows, fVecDim, bVecDim, f16VecDim)
idata, err := InsertMsgToInsertData(msg, schema) idata, err := InsertMsgToInsertData(msg, schema)
assert.NoError(t, err) assert.NoError(t, err)

View File

@ -524,6 +524,49 @@ func (p *BinlogFile) ReadBinaryVector() ([]byte, int, error) {
return result, dim, nil return result, dim, nil
} }
func (p *BinlogFile) ReadFloat16Vector() ([]byte, int, error) {
if p.reader == nil {
log.Warn("Binlog file: binlog reader not yet initialized")
return nil, 0, errors.New("binlog reader not yet initialized")
}
dim := 0
result := make([]byte, 0)
for {
event, err := p.reader.NextEventReader()
if err != nil {
log.Warn("Binlog file: failed to iterate events reader", zap.Error(err))
return nil, 0, fmt.Errorf("failed to iterate events reader, error: %w", err)
}
// end of the file
if event == nil {
break
}
if event.TypeCode != storage.InsertEventType {
log.Warn("Binlog file: binlog file is not insert log")
return nil, 0, errors.New("binlog file is not insert log")
}
if p.DataType() != schemapb.DataType_Float16Vector {
log.Warn("Binlog file: binlog data type is not float16 vector")
return nil, 0, errors.New("binlog data type is not float16 vector")
}
data, dimenson, err := event.PayloadReaderInterface.GetFloat16VectorFromPayload()
if err != nil {
log.Warn("Binlog file: failed to read float16 vector data", zap.Error(err))
return nil, 0, fmt.Errorf("failed to read float16 vector data, error: %w", err)
}
dim = dimenson
result = append(result, data...)
}
return result, dim, nil
}
// ReadFloatVector method reads all the blocks of a binlog by a data type. // ReadFloatVector method reads all the blocks of a binlog by a data type.
// A binlog is designed to support multiple blocks, but so far each binlog always contains only one block. // A binlog is designed to support multiple blocks, but so far each binlog always contains only one block.
// return vectors data and the dimension // return vectors data and the dimension

View File

@ -43,6 +43,11 @@ func createBinlogBuf(t *testing.T, dataType schemapb.DataType, data interface{})
if len(vectors) > 0 { if len(vectors) > 0 {
dim = len(vectors[0]) dim = len(vectors[0])
} }
} else if dataType == schemapb.DataType_Float16Vector {
vectors := data.([][]byte)
if len(vectors) > 0 {
dim = len(vectors[0]) / 2
}
} }
evt, err := w.NextInsertEventWriter(dim) evt, err := w.NextInsertEventWriter(dim)
@ -144,6 +149,16 @@ func createBinlogBuf(t *testing.T, dataType schemapb.DataType, data interface{})
// the "original_size" is come from storage.originalSizeKey // the "original_size" is come from storage.originalSizeKey
sizeTotal := len(vectors) * dim * 4 sizeTotal := len(vectors) * dim * 4
w.AddExtra("original_size", fmt.Sprintf("%v", sizeTotal)) w.AddExtra("original_size", fmt.Sprintf("%v", sizeTotal))
case schemapb.DataType_Float16Vector:
vectors := data.([][]byte)
for i := 0; i < len(vectors); i++ {
err = evt.AddFloat16VectorToPayload(vectors[i], dim)
assert.NoError(t, err)
}
// without the two lines, the case will crash at here.
// the "original_size" is come from storage.originalSizeKey
sizeTotal := len(vectors) * dim * 2
w.AddExtra("original_size", fmt.Sprintf("%v", sizeTotal))
default: default:
assert.True(t, false) assert.True(t, false)
return nil return nil
@ -256,6 +271,11 @@ func Test_BinlogFileOpen(t *testing.T) {
assert.Nil(t, dataFloatVector) assert.Nil(t, dataFloatVector)
assert.Equal(t, 0, dim) assert.Equal(t, 0, dim)
assert.Error(t, err) assert.Error(t, err)
dataFloat16Vector, dim, err := binlogFile.ReadFloat16Vector()
assert.Nil(t, dataFloat16Vector)
assert.Equal(t, 0, dim)
assert.Error(t, err)
} }
func Test_BinlogFileBool(t *testing.T) { func Test_BinlogFileBool(t *testing.T) {
@ -894,3 +914,71 @@ func Test_BinlogFileFloatVector(t *testing.T) {
binlogFile.Close() binlogFile.Close()
} }
func Test_BinlogFileFloat16Vector(t *testing.T) {
vectors := make([][]byte, 0)
vectors = append(vectors, []byte{1, 3, 5, 7})
vectors = append(vectors, []byte{2, 4, 6, 8})
dim := len(vectors[0]) / 2
vecCount := len(vectors)
chunkManager := &MockChunkManager{
readBuf: map[string][]byte{
"dummy": createBinlogBuf(t, schemapb.DataType_Float16Vector, vectors),
},
}
binlogFile, err := NewBinlogFile(chunkManager)
assert.NoError(t, err)
assert.NotNil(t, binlogFile)
// correct reading
err = binlogFile.Open("dummy")
assert.NoError(t, err)
assert.Equal(t, schemapb.DataType_Float16Vector, binlogFile.DataType())
data, d, err := binlogFile.ReadFloat16Vector()
assert.NoError(t, err)
assert.Equal(t, dim, d)
assert.NotNil(t, data)
assert.Equal(t, vecCount*dim*2, len(data))
for i := 0; i < vecCount; i++ {
for j := 0; j < dim*2; j++ {
assert.Equal(t, vectors[i][j], data[i*dim*2+j])
}
}
binlogFile.Close()
// wrong data type reading
binlogFile, err = NewBinlogFile(chunkManager)
assert.NoError(t, err)
err = binlogFile.Open("dummy")
assert.NoError(t, err)
dt, d, err := binlogFile.ReadFloatVector()
assert.Zero(t, len(dt))
assert.Zero(t, d)
assert.Error(t, err)
binlogFile.Close()
// wrong log type
chunkManager.readBuf["dummy"] = createDeltalogBuf(t, []int64{1}, false)
err = binlogFile.Open("dummy")
assert.NoError(t, err)
data, d, err = binlogFile.ReadFloat16Vector()
assert.Zero(t, len(data))
assert.Zero(t, d)
assert.Error(t, err)
// failed to iterate events reader
binlogFile.reader.Close()
data, d, err = binlogFile.ReadFloat16Vector()
assert.Zero(t, len(data))
assert.Zero(t, d)
assert.Error(t, err)
binlogFile.Close()
}

View File

@ -115,6 +115,8 @@ func (index *CgoIndex) Build(dataset *Dataset) error {
return fmt.Errorf("build index on supported data type: %s", dataset.DType.String()) return fmt.Errorf("build index on supported data type: %s", dataset.DType.String())
case schemapb.DataType_FloatVector: case schemapb.DataType_FloatVector:
return index.buildFloatVecIndex(dataset) return index.buildFloatVecIndex(dataset)
case schemapb.DataType_Float16Vector:
return fmt.Errorf("build index on supported data type: %s", dataset.DType.String())
case schemapb.DataType_BinaryVector: case schemapb.DataType_BinaryVector:
return index.buildBinaryVecIndex(dataset) return index.buildBinaryVecIndex(dataset)
case schemapb.DataType_Bool: case schemapb.DataType_Bool:

View File

@ -67,6 +67,7 @@ func TestGenEmptyFieldData(t *testing.T) {
vectorTypes := []schemapb.DataType{ vectorTypes := []schemapb.DataType{
schemapb.DataType_BinaryVector, schemapb.DataType_BinaryVector,
schemapb.DataType_FloatVector, schemapb.DataType_FloatVector,
schemapb.DataType_Float16Vector,
} }
field := &schemapb.FieldSchema{Name: "field_name", FieldID: 100} field := &schemapb.FieldSchema{Name: "field_name", FieldID: 100}

View File

@ -13,7 +13,7 @@ require (
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
github.com/klauspost/compress v1.16.5 github.com/klauspost/compress v1.16.5
github.com/lingdor/stackerror v0.0.0-20191119040541-976d8885ed76 github.com/lingdor/stackerror v0.0.0-20191119040541-976d8885ed76
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.0-dev.1.0.20230716112827-c3fe148f5e1d github.com/milvus-io/milvus-proto/go-api/v2 v2.3.0
github.com/nats-io/nats-server/v2 v2.9.17 github.com/nats-io/nats-server/v2 v2.9.17
github.com/nats-io/nats.go v1.24.0 github.com/nats-io/nats.go v1.24.0
github.com/panjf2000/ants/v2 v2.7.2 github.com/panjf2000/ants/v2 v2.7.2

View File

@ -477,8 +477,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfr
github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8= github.com/mediocregopher/radix/v3 v3.4.2/go.mod h1:8FL3F6UQRXHXIBSPUs5h0RybMF8i4n7wVopoX3x7Bv8=
github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc= github.com/microcosm-cc/bluemonday v1.0.2/go.mod h1:iVP4YcDBq+n/5fb23BhYFvIMq/leAFZyRl6bYmGDlGc=
github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.0-dev.1.0.20230716112827-c3fe148f5e1d h1:XsQQ/MigebXEE2VXPKKmA3K7OHC+mkEUiErWvaWMikI= github.com/milvus-io/milvus-proto/go-api/v2 v2.3.0 h1:t5CKm7+FXuD2rDLv/H8tpN9iY8F2dZvHF87xWBx8muU=
github.com/milvus-io/milvus-proto/go-api/v2 v2.3.0-dev.1.0.20230716112827-c3fe148f5e1d/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek= github.com/milvus-io/milvus-proto/go-api/v2 v2.3.0/go.mod h1:1OIl0v5PQeNxIJhCvY+K55CBUOYDZevw9g9380u1Wek=
github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A= github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A=
github.com/milvus-io/pulsar-client-go v0.6.10/go.mod h1:lQqCkgwDF8YFYjKA+zOheTk1tev2B+bKj5j7+nm8M1w= github.com/milvus-io/pulsar-client-go v0.6.10/go.mod h1:lQqCkgwDF8YFYjKA+zOheTk1tev2B+bKj5j7+nm8M1w=
github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=

View File

@ -108,7 +108,7 @@ func CheckCtxValid(ctx context.Context) bool {
func GetVecFieldIDs(schema *schemapb.CollectionSchema) []int64 { func GetVecFieldIDs(schema *schemapb.CollectionSchema) []int64 {
var vecFieldIDs []int64 var vecFieldIDs []int64
for _, field := range schema.Fields { for _, field := range schema.Fields {
if field.DataType == schemapb.DataType_BinaryVector || field.DataType == schemapb.DataType_FloatVector { if field.DataType == schemapb.DataType_BinaryVector || field.DataType == schemapb.DataType_FloatVector || field.DataType == schemapb.DataType_Float16Vector {
vecFieldIDs = append(vecFieldIDs, field.FieldID) vecFieldIDs = append(vecFieldIDs, field.FieldID)
} }
} }
@ -222,7 +222,17 @@ func GetNumRowsOfBinaryVectorField(bDatas []byte, dim int64) (uint64, error) {
return uint64((8 * int64(l)) / dim), nil return uint64((8 * int64(l)) / dim), nil
} }
// GetNumRowOfFieldData return num rows of the field data func GetNumRowsOfFloat16VectorField(f16Datas []byte, dim int64) (uint64, error) {
if dim <= 0 {
return 0, fmt.Errorf("dim(%d) should be greater than 0", dim)
}
l := len(f16Datas)
if int64(l)%dim != 0 {
return 0, fmt.Errorf("the length(%d) of float data should divide the dim(%d)", l, dim)
}
return uint64((int64(l)) / dim / 2), nil
}
func GetNumRowOfFieldData(fieldData *schemapb.FieldData) (uint64, error) { func GetNumRowOfFieldData(fieldData *schemapb.FieldData) (uint64, error) {
var fieldNumRows uint64 var fieldNumRows uint64
var err error var err error
@ -264,6 +274,12 @@ func GetNumRowOfFieldData(fieldData *schemapb.FieldData) (uint64, error) {
if err != nil { if err != nil {
return 0, err return 0, err
} }
case *schemapb.VectorField_Float16Vector:
dim := vectorField.GetDim()
fieldNumRows, err = GetNumRowsOfFloat16VectorField(vectorField.GetFloat16Vector(), dim)
if err != nil {
return 0, err
}
default: default:
return 0, fmt.Errorf("%s is not supported now", vectorFieldType) return 0, fmt.Errorf("%s is not supported now", vectorFieldType)
} }

View File

@ -219,6 +219,34 @@ func TestGetNumRowsOfFloatVectorField(t *testing.T) {
} }
} }
func TestGetNumRowsOfFloat16VectorField(t *testing.T) {
cases := []struct {
bDatas []byte
dim int64
want uint64
errIsNil bool
}{
{[]byte{}, -1, 0, false}, // dim <= 0
{[]byte{}, 0, 0, false}, // dim <= 0
{[]byte{1.0}, 128, 0, false}, // length % dim != 0
{[]byte{}, 128, 0, true},
{[]byte{1.0, 2.0}, 1, 1, true},
{[]byte{1.0, 2.0, 3.0, 4.0}, 2, 1, true},
}
for _, test := range cases {
got, err := GetNumRowsOfFloat16VectorField(test.bDatas, test.dim)
if test.errIsNil {
assert.Equal(t, nil, err)
if got != test.want {
t.Errorf("GetNumRowsOfFloat16VectorField(%v, %v) = %v, %v", test.bDatas, test.dim, test.want, nil)
}
} else {
assert.NotEqual(t, nil, err)
}
}
}
func TestGetNumRowsOfBinaryVectorField(t *testing.T) { func TestGetNumRowsOfBinaryVectorField(t *testing.T) {
cases := []struct { cases := []struct {
bDatas []byte bDatas []byte

View File

@ -29,10 +29,11 @@ func (c floatVectorBaseChecker) CheckTrain(params map[string]string) error {
} }
func (c floatVectorBaseChecker) CheckValidDataType(dType schemapb.DataType) error { func (c floatVectorBaseChecker) CheckValidDataType(dType schemapb.DataType) error {
if dType != schemapb.DataType_FloatVector { if dType == schemapb.DataType_FloatVector || dType == schemapb.DataType_Float16Vector {
return fmt.Errorf("float vector is only supported") return nil
} else {
return fmt.Errorf("float or float16 vector are only supported")
} }
return nil
} }
func (c floatVectorBaseChecker) SetDefaultMetricTypeIfNotExist(params map[string]string) { func (c floatVectorBaseChecker) SetDefaultMetricTypeIfNotExist(params map[string]string) {

View File

@ -31,7 +31,7 @@ func (c hnswChecker) CheckTrain(params map[string]string) error {
} }
func (c hnswChecker) CheckValidDataType(dType schemapb.DataType) error { func (c hnswChecker) CheckValidDataType(dType schemapb.DataType) error {
if dType != schemapb.DataType_FloatVector && dType != schemapb.DataType_BinaryVector { if dType != schemapb.DataType_FloatVector && dType != schemapb.DataType_BinaryVector && dType != schemapb.DataType_Float16Vector {
return fmt.Errorf("only support float vector or binary vector") return fmt.Errorf("only support float vector or binary vector")
} }
return nil return nil

View File

@ -150,6 +150,26 @@ func genEmptyFloatVectorFieldData(field *schemapb.FieldSchema) (*schemapb.FieldD
}, nil }, nil
} }
func genEmptyFloat16VectorFieldData(field *schemapb.FieldSchema) (*schemapb.FieldData, error) {
dim, err := GetDim(field)
if err != nil {
return nil, err
}
return &schemapb.FieldData{
Type: field.GetDataType(),
FieldName: field.GetName(),
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: dim,
Data: &schemapb.VectorField_Float16Vector{
Float16Vector: nil,
},
},
},
FieldId: field.GetFieldID(),
}, nil
}
func GenEmptyFieldData(field *schemapb.FieldSchema) (*schemapb.FieldData, error) { func GenEmptyFieldData(field *schemapb.FieldSchema) (*schemapb.FieldData, error) {
dataType := field.GetDataType() dataType := field.GetDataType()
switch dataType { switch dataType {
@ -173,6 +193,8 @@ func GenEmptyFieldData(field *schemapb.FieldSchema) (*schemapb.FieldData, error)
return genEmptyBinaryVectorFieldData(field) return genEmptyBinaryVectorFieldData(field)
case schemapb.DataType_FloatVector: case schemapb.DataType_FloatVector:
return genEmptyFloatVectorFieldData(field) return genEmptyFloatVectorFieldData(field)
case schemapb.DataType_Float16Vector:
return genEmptyFloat16VectorFieldData(field)
default: default:
return nil, fmt.Errorf("unsupported data type: %s", dataType.String()) return nil, fmt.Errorf("unsupported data type: %s", dataType.String())
} }

View File

@ -105,6 +105,17 @@ func EstimateSizePerRecord(schema *schemapb.CollectionSchema) (int, error) {
break break
} }
} }
case schemapb.DataType_Float16Vector:
for _, kv := range fs.TypeParams {
if kv.Key == common.DimKey {
v, err := strconv.Atoi(kv.Value)
if err != nil {
return -1, err
}
res += v * 2
break
}
}
} }
} }
return res, nil return res, nil
@ -305,7 +316,7 @@ func (helper *SchemaHelper) GetVectorDimFromID(fieldID int64) (int, error) {
// IsVectorType returns true if input is a vector type, otherwise false // IsVectorType returns true if input is a vector type, otherwise false
func IsVectorType(dataType schemapb.DataType) bool { func IsVectorType(dataType schemapb.DataType) bool {
switch dataType { switch dataType {
case schemapb.DataType_FloatVector, schemapb.DataType_BinaryVector: case schemapb.DataType_FloatVector, schemapb.DataType_BinaryVector, schemapb.DataType_Float16Vector:
return true return true
default: default:
return false return false
@ -510,6 +521,17 @@ func AppendFieldData(dst []*schemapb.FieldData, src []*schemapb.FieldData, idx i
} else { } else {
dstVector.GetFloatVector().Data = append(dstVector.GetFloatVector().Data, srcVector.FloatVector.Data[idx*dim:(idx+1)*dim]...) dstVector.GetFloatVector().Data = append(dstVector.GetFloatVector().Data, srcVector.FloatVector.Data[idx*dim:(idx+1)*dim]...)
} }
case *schemapb.VectorField_Float16Vector:
if dstVector.GetFloat16Vector() == nil {
srcToCopy := srcVector.Float16Vector[idx*(dim*2) : (idx+1)*(dim*2)]
dstVector.Data = &schemapb.VectorField_Float16Vector{
Float16Vector: make([]byte, len(srcToCopy)),
}
copy(dstVector.Data.(*schemapb.VectorField_Float16Vector).Float16Vector, srcToCopy)
} else {
dstFloat16Vector := dstVector.Data.(*schemapb.VectorField_Float16Vector)
dstFloat16Vector.Float16Vector = append(dstFloat16Vector.Float16Vector, srcVector.Float16Vector[idx*(dim*2):(idx+1)*(dim*2)]...)
}
default: default:
log.Error("Not supported field type", zap.String("field type", fieldData.Type.String())) log.Error("Not supported field type", zap.String("field type", fieldData.Type.String()))
} }
@ -558,6 +580,9 @@ func DeleteFieldData(dst []*schemapb.FieldData) {
dstBinaryVector.BinaryVector = dstBinaryVector.BinaryVector[:len(dstBinaryVector.BinaryVector)-int(dim/8)] dstBinaryVector.BinaryVector = dstBinaryVector.BinaryVector[:len(dstBinaryVector.BinaryVector)-int(dim/8)]
case *schemapb.VectorField_FloatVector: case *schemapb.VectorField_FloatVector:
dstVector.GetFloatVector().Data = dstVector.GetFloatVector().Data[:len(dstVector.GetFloatVector().Data)-int(dim)] dstVector.GetFloatVector().Data = dstVector.GetFloatVector().Data[:len(dstVector.GetFloatVector().Data)-int(dim)]
case *schemapb.VectorField_Float16Vector:
dstFloat16Vector := dstVector.Data.(*schemapb.VectorField_Float16Vector)
dstFloat16Vector.Float16Vector = dstFloat16Vector.Float16Vector[:len(dstFloat16Vector.Float16Vector)-int(dim*2)]
default: default:
log.Error("wrong field type added", zap.String("field type", fieldData.Type.String())) log.Error("wrong field type added", zap.String("field type", fieldData.Type.String()))
} }
@ -873,6 +898,10 @@ func GetData(field *schemapb.FieldData, idx int) interface{} {
dim := int(field.GetVectors().GetDim()) dim := int(field.GetVectors().GetDim())
dataBytes := dim / 8 dataBytes := dim / 8
return field.GetVectors().GetBinaryVector()[idx*dataBytes : (idx+1)*dataBytes] return field.GetVectors().GetBinaryVector()[idx*dataBytes : (idx+1)*dataBytes]
case schemapb.DataType_Float16Vector:
dim := int(field.GetVectors().GetDim())
dataBytes := dim * 2
return field.GetVectors().GetFloat16Vector()[idx*dataBytes : (idx+1)*dataBytes]
} }
return nil return nil
} }

View File

@ -588,6 +588,20 @@ func genFieldData(fieldName string, fieldID int64, fieldType schemapb.DataType,
}, },
FieldId: fieldID, FieldId: fieldID,
} }
case schemapb.DataType_Float16Vector:
fieldData = &schemapb.FieldData{
Type: schemapb.DataType_Float16Vector,
FieldName: fieldName,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: dim,
Data: &schemapb.VectorField_Float16Vector{
Float16Vector: fieldValue.([]byte),
},
},
},
FieldId: fieldID,
}
case schemapb.DataType_Array: case schemapb.DataType_Array:
data := fieldValue.([][]int32) data := fieldValue.([][]int32)
fieldData = &schemapb.FieldData{ fieldData = &schemapb.FieldData{
@ -640,21 +654,23 @@ func genFieldData(fieldName string, fieldID int64, fieldType schemapb.DataType,
func TestAppendFieldData(t *testing.T) { func TestAppendFieldData(t *testing.T) {
const ( const (
Dim = 8 Dim = 8
BoolFieldName = "BoolField" BoolFieldName = "BoolField"
Int32FieldName = "Int32Field" Int32FieldName = "Int32Field"
Int64FieldName = "Int64Field" Int64FieldName = "Int64Field"
FloatFieldName = "FloatField" FloatFieldName = "FloatField"
DoubleFieldName = "DoubleField" DoubleFieldName = "DoubleField"
BinaryVectorFieldName = "BinaryVectorField" BinaryVectorFieldName = "BinaryVectorField"
FloatVectorFieldName = "FloatVectorField" FloatVectorFieldName = "FloatVectorField"
BoolFieldID = common.StartOfUserFieldID + 1 Float16VectorFieldName = "Float16VectorField"
Int32FieldID = common.StartOfUserFieldID + 2 BoolFieldID = common.StartOfUserFieldID + 1
Int64FieldID = common.StartOfUserFieldID + 3 Int32FieldID = common.StartOfUserFieldID + 2
FloatFieldID = common.StartOfUserFieldID + 4 Int64FieldID = common.StartOfUserFieldID + 3
DoubleFieldID = common.StartOfUserFieldID + 5 FloatFieldID = common.StartOfUserFieldID + 4
BinaryVectorFieldID = common.StartOfUserFieldID + 6 DoubleFieldID = common.StartOfUserFieldID + 5
FloatVectorFieldID = common.StartOfUserFieldID + 7 BinaryVectorFieldID = common.StartOfUserFieldID + 6
FloatVectorFieldID = common.StartOfUserFieldID + 7
Float16VectorFieldID = common.StartOfUserFieldID + 8
) )
BoolArray := []bool{true, false} BoolArray := []bool{true, false}
Int32Array := []int32{1, 2} Int32Array := []int32{1, 2}
@ -663,8 +679,10 @@ func TestAppendFieldData(t *testing.T) {
DoubleArray := []float64{11.0, 22.0} DoubleArray := []float64{11.0, 22.0}
BinaryVector := []byte{0x12, 0x34} BinaryVector := []byte{0x12, 0x34}
FloatVector := []float32{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 11.0, 22.0, 33.0, 44.0, 55.0, 66.0, 77.0, 88.0} FloatVector := []float32{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 11.0, 22.0, 33.0, 44.0, 55.0, 66.0, 77.0, 88.0}
Float16Vector := []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}
result := make([]*schemapb.FieldData, 7) result := make([]*schemapb.FieldData, 8)
var fieldDataArray1 []*schemapb.FieldData var fieldDataArray1 []*schemapb.FieldData
fieldDataArray1 = append(fieldDataArray1, genFieldData(BoolFieldName, BoolFieldID, schemapb.DataType_Bool, BoolArray[0:1], 1)) fieldDataArray1 = append(fieldDataArray1, genFieldData(BoolFieldName, BoolFieldID, schemapb.DataType_Bool, BoolArray[0:1], 1))
fieldDataArray1 = append(fieldDataArray1, genFieldData(Int32FieldName, Int32FieldID, schemapb.DataType_Int32, Int32Array[0:1], 1)) fieldDataArray1 = append(fieldDataArray1, genFieldData(Int32FieldName, Int32FieldID, schemapb.DataType_Int32, Int32Array[0:1], 1))
@ -673,6 +691,7 @@ func TestAppendFieldData(t *testing.T) {
fieldDataArray1 = append(fieldDataArray1, genFieldData(DoubleFieldName, DoubleFieldID, schemapb.DataType_Double, DoubleArray[0:1], 1)) fieldDataArray1 = append(fieldDataArray1, genFieldData(DoubleFieldName, DoubleFieldID, schemapb.DataType_Double, DoubleArray[0:1], 1))
fieldDataArray1 = append(fieldDataArray1, genFieldData(BinaryVectorFieldName, BinaryVectorFieldID, schemapb.DataType_BinaryVector, BinaryVector[0:Dim/8], Dim)) fieldDataArray1 = append(fieldDataArray1, genFieldData(BinaryVectorFieldName, BinaryVectorFieldID, schemapb.DataType_BinaryVector, BinaryVector[0:Dim/8], Dim))
fieldDataArray1 = append(fieldDataArray1, genFieldData(FloatVectorFieldName, FloatVectorFieldID, schemapb.DataType_FloatVector, FloatVector[0:Dim], Dim)) fieldDataArray1 = append(fieldDataArray1, genFieldData(FloatVectorFieldName, FloatVectorFieldID, schemapb.DataType_FloatVector, FloatVector[0:Dim], Dim))
fieldDataArray1 = append(fieldDataArray1, genFieldData(Float16VectorFieldName, Float16VectorFieldID, schemapb.DataType_Float16Vector, Float16Vector[0:Dim*2], Dim))
var fieldDataArray2 []*schemapb.FieldData var fieldDataArray2 []*schemapb.FieldData
fieldDataArray2 = append(fieldDataArray2, genFieldData(BoolFieldName, BoolFieldID, schemapb.DataType_Bool, BoolArray[1:2], 1)) fieldDataArray2 = append(fieldDataArray2, genFieldData(BoolFieldName, BoolFieldID, schemapb.DataType_Bool, BoolArray[1:2], 1))
@ -682,6 +701,7 @@ func TestAppendFieldData(t *testing.T) {
fieldDataArray2 = append(fieldDataArray2, genFieldData(DoubleFieldName, DoubleFieldID, schemapb.DataType_Double, DoubleArray[1:2], 1)) fieldDataArray2 = append(fieldDataArray2, genFieldData(DoubleFieldName, DoubleFieldID, schemapb.DataType_Double, DoubleArray[1:2], 1))
fieldDataArray2 = append(fieldDataArray2, genFieldData(BinaryVectorFieldName, BinaryVectorFieldID, schemapb.DataType_BinaryVector, BinaryVector[Dim/8:2*Dim/8], Dim)) fieldDataArray2 = append(fieldDataArray2, genFieldData(BinaryVectorFieldName, BinaryVectorFieldID, schemapb.DataType_BinaryVector, BinaryVector[Dim/8:2*Dim/8], Dim))
fieldDataArray2 = append(fieldDataArray2, genFieldData(FloatVectorFieldName, FloatVectorFieldID, schemapb.DataType_FloatVector, FloatVector[Dim:2*Dim], Dim)) fieldDataArray2 = append(fieldDataArray2, genFieldData(FloatVectorFieldName, FloatVectorFieldID, schemapb.DataType_FloatVector, FloatVector[Dim:2*Dim], Dim))
fieldDataArray2 = append(fieldDataArray2, genFieldData(Float16VectorFieldName, Float16VectorFieldID, schemapb.DataType_Float16Vector, Float16Vector[2*Dim:4*Dim], Dim))
AppendFieldData(result, fieldDataArray1, 0) AppendFieldData(result, fieldDataArray1, 0)
AppendFieldData(result, fieldDataArray2, 0) AppendFieldData(result, fieldDataArray2, 0)
@ -693,19 +713,21 @@ func TestAppendFieldData(t *testing.T) {
assert.Equal(t, DoubleArray, result[4].GetScalars().GetDoubleData().Data) assert.Equal(t, DoubleArray, result[4].GetScalars().GetDoubleData().Data)
assert.Equal(t, BinaryVector, result[5].GetVectors().Data.(*schemapb.VectorField_BinaryVector).BinaryVector) assert.Equal(t, BinaryVector, result[5].GetVectors().Data.(*schemapb.VectorField_BinaryVector).BinaryVector)
assert.Equal(t, FloatVector, result[6].GetVectors().GetFloatVector().Data) assert.Equal(t, FloatVector, result[6].GetVectors().GetFloatVector().Data)
assert.Equal(t, Float16Vector, result[7].GetVectors().Data.(*schemapb.VectorField_Float16Vector).Float16Vector)
} }
func TestDeleteFieldData(t *testing.T) { func TestDeleteFieldData(t *testing.T) {
const ( const (
Dim = 8 Dim = 8
BoolFieldName = "BoolField" BoolFieldName = "BoolField"
Int32FieldName = "Int32Field" Int32FieldName = "Int32Field"
Int64FieldName = "Int64Field" Int64FieldName = "Int64Field"
FloatFieldName = "FloatField" FloatFieldName = "FloatField"
DoubleFieldName = "DoubleField" DoubleFieldName = "DoubleField"
JSONFieldName = "JSONField" JSONFieldName = "JSONField"
BinaryVectorFieldName = "BinaryVectorField" BinaryVectorFieldName = "BinaryVectorField"
FloatVectorFieldName = "FloatVectorField" FloatVectorFieldName = "FloatVectorField"
Float16VectorFieldName = "Float16VectorField"
) )
const ( const (
@ -717,6 +739,7 @@ func TestDeleteFieldData(t *testing.T) {
JSONFieldID JSONFieldID
BinaryVectorFieldID BinaryVectorFieldID
FloatVectorFieldID FloatVectorFieldID
Float16VectorFieldID
) )
BoolArray := []bool{true, false} BoolArray := []bool{true, false}
Int32Array := []int32{1, 2} Int32Array := []int32{1, 2}
@ -726,9 +749,11 @@ func TestDeleteFieldData(t *testing.T) {
JSONArray := [][]byte{[]byte("{\"hello\":0}"), []byte("{\"key\":1}")} JSONArray := [][]byte{[]byte("{\"hello\":0}"), []byte("{\"key\":1}")}
BinaryVector := []byte{0x12, 0x34} BinaryVector := []byte{0x12, 0x34}
FloatVector := []float32{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 11.0, 22.0, 33.0, 44.0, 55.0, 66.0, 77.0, 88.0} FloatVector := []float32{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 11.0, 22.0, 33.0, 44.0, 55.0, 66.0, 77.0, 88.0}
Float16Vector := []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff,
0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x88, 0x99, 0xaa, 0xbb, 0xcc, 0xdd, 0xee, 0xff}
result1 := make([]*schemapb.FieldData, 8) result1 := make([]*schemapb.FieldData, 9)
result2 := make([]*schemapb.FieldData, 8) result2 := make([]*schemapb.FieldData, 9)
var fieldDataArray1 []*schemapb.FieldData var fieldDataArray1 []*schemapb.FieldData
fieldDataArray1 = append(fieldDataArray1, genFieldData(BoolFieldName, BoolFieldID, schemapb.DataType_Bool, BoolArray[0:1], 1)) fieldDataArray1 = append(fieldDataArray1, genFieldData(BoolFieldName, BoolFieldID, schemapb.DataType_Bool, BoolArray[0:1], 1))
fieldDataArray1 = append(fieldDataArray1, genFieldData(Int32FieldName, Int32FieldID, schemapb.DataType_Int32, Int32Array[0:1], 1)) fieldDataArray1 = append(fieldDataArray1, genFieldData(Int32FieldName, Int32FieldID, schemapb.DataType_Int32, Int32Array[0:1], 1))
@ -738,6 +763,7 @@ func TestDeleteFieldData(t *testing.T) {
fieldDataArray1 = append(fieldDataArray1, genFieldData(JSONFieldName, JSONFieldID, schemapb.DataType_JSON, JSONArray[0:1], 1)) fieldDataArray1 = append(fieldDataArray1, genFieldData(JSONFieldName, JSONFieldID, schemapb.DataType_JSON, JSONArray[0:1], 1))
fieldDataArray1 = append(fieldDataArray1, genFieldData(BinaryVectorFieldName, BinaryVectorFieldID, schemapb.DataType_BinaryVector, BinaryVector[0:Dim/8], Dim)) fieldDataArray1 = append(fieldDataArray1, genFieldData(BinaryVectorFieldName, BinaryVectorFieldID, schemapb.DataType_BinaryVector, BinaryVector[0:Dim/8], Dim))
fieldDataArray1 = append(fieldDataArray1, genFieldData(FloatVectorFieldName, FloatVectorFieldID, schemapb.DataType_FloatVector, FloatVector[0:Dim], Dim)) fieldDataArray1 = append(fieldDataArray1, genFieldData(FloatVectorFieldName, FloatVectorFieldID, schemapb.DataType_FloatVector, FloatVector[0:Dim], Dim))
fieldDataArray1 = append(fieldDataArray1, genFieldData(Float16VectorFieldName, Float16VectorFieldID, schemapb.DataType_Float16Vector, Float16Vector[0:2*Dim], Dim))
var fieldDataArray2 []*schemapb.FieldData var fieldDataArray2 []*schemapb.FieldData
fieldDataArray2 = append(fieldDataArray2, genFieldData(BoolFieldName, BoolFieldID, schemapb.DataType_Bool, BoolArray[1:2], 1)) fieldDataArray2 = append(fieldDataArray2, genFieldData(BoolFieldName, BoolFieldID, schemapb.DataType_Bool, BoolArray[1:2], 1))
@ -748,6 +774,7 @@ func TestDeleteFieldData(t *testing.T) {
fieldDataArray2 = append(fieldDataArray2, genFieldData(JSONFieldName, JSONFieldID, schemapb.DataType_JSON, JSONArray[1:2], 1)) fieldDataArray2 = append(fieldDataArray2, genFieldData(JSONFieldName, JSONFieldID, schemapb.DataType_JSON, JSONArray[1:2], 1))
fieldDataArray2 = append(fieldDataArray2, genFieldData(BinaryVectorFieldName, BinaryVectorFieldID, schemapb.DataType_BinaryVector, BinaryVector[Dim/8:2*Dim/8], Dim)) fieldDataArray2 = append(fieldDataArray2, genFieldData(BinaryVectorFieldName, BinaryVectorFieldID, schemapb.DataType_BinaryVector, BinaryVector[Dim/8:2*Dim/8], Dim))
fieldDataArray2 = append(fieldDataArray2, genFieldData(FloatVectorFieldName, FloatVectorFieldID, schemapb.DataType_FloatVector, FloatVector[Dim:2*Dim], Dim)) fieldDataArray2 = append(fieldDataArray2, genFieldData(FloatVectorFieldName, FloatVectorFieldID, schemapb.DataType_FloatVector, FloatVector[Dim:2*Dim], Dim))
fieldDataArray2 = append(fieldDataArray2, genFieldData(Float16VectorFieldName, Float16VectorFieldID, schemapb.DataType_Float16Vector, Float16Vector[2*Dim:4*Dim], Dim))
AppendFieldData(result1, fieldDataArray1, 0) AppendFieldData(result1, fieldDataArray1, 0)
AppendFieldData(result1, fieldDataArray2, 0) AppendFieldData(result1, fieldDataArray2, 0)
@ -760,6 +787,7 @@ func TestDeleteFieldData(t *testing.T) {
assert.Equal(t, JSONArray[0:1], result1[JSONFieldID-common.StartOfUserFieldID].GetScalars().GetJsonData().Data) assert.Equal(t, JSONArray[0:1], result1[JSONFieldID-common.StartOfUserFieldID].GetScalars().GetJsonData().Data)
assert.Equal(t, BinaryVector[0:Dim/8], result1[BinaryVectorFieldID-common.StartOfUserFieldID].GetVectors().Data.(*schemapb.VectorField_BinaryVector).BinaryVector) assert.Equal(t, BinaryVector[0:Dim/8], result1[BinaryVectorFieldID-common.StartOfUserFieldID].GetVectors().Data.(*schemapb.VectorField_BinaryVector).BinaryVector)
assert.Equal(t, FloatVector[0:Dim], result1[FloatVectorFieldID-common.StartOfUserFieldID].GetVectors().GetFloatVector().Data) assert.Equal(t, FloatVector[0:Dim], result1[FloatVectorFieldID-common.StartOfUserFieldID].GetVectors().GetFloatVector().Data)
assert.Equal(t, Float16Vector[0:2*Dim], result1[Float16VectorFieldID-common.StartOfUserFieldID].GetVectors().Data.(*schemapb.VectorField_Float16Vector).Float16Vector)
AppendFieldData(result2, fieldDataArray2, 0) AppendFieldData(result2, fieldDataArray2, 0)
AppendFieldData(result2, fieldDataArray1, 0) AppendFieldData(result2, fieldDataArray1, 0)
@ -772,6 +800,7 @@ func TestDeleteFieldData(t *testing.T) {
assert.Equal(t, JSONArray[1:2], result2[JSONFieldID-common.StartOfUserFieldID].GetScalars().GetJsonData().Data) assert.Equal(t, JSONArray[1:2], result2[JSONFieldID-common.StartOfUserFieldID].GetScalars().GetJsonData().Data)
assert.Equal(t, BinaryVector[Dim/8:2*Dim/8], result2[BinaryVectorFieldID-common.StartOfUserFieldID].GetVectors().Data.(*schemapb.VectorField_BinaryVector).BinaryVector) assert.Equal(t, BinaryVector[Dim/8:2*Dim/8], result2[BinaryVectorFieldID-common.StartOfUserFieldID].GetVectors().Data.(*schemapb.VectorField_BinaryVector).BinaryVector)
assert.Equal(t, FloatVector[Dim:2*Dim], result2[FloatVectorFieldID-common.StartOfUserFieldID].GetVectors().GetFloatVector().Data) assert.Equal(t, FloatVector[Dim:2*Dim], result2[FloatVectorFieldID-common.StartOfUserFieldID].GetVectors().GetFloatVector().Data)
assert.Equal(t, Float16Vector[2*Dim:4*Dim], result2[Float16VectorFieldID-common.StartOfUserFieldID].GetVectors().Data.(*schemapb.VectorField_Float16Vector).Float16Vector)
} }
func TestGetPrimaryFieldSchema(t *testing.T) { func TestGetPrimaryFieldSchema(t *testing.T) {
@ -1091,6 +1120,8 @@ func TestGetDataAndGetDataSize(t *testing.T) {
VarCharArray := []string{"a", "b"} VarCharArray := []string{"a", "b"}
BinaryVector := []byte{0x12, 0x34} BinaryVector := []byte{0x12, 0x34}
FloatVector := []float32{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 11.0, 22.0, 33.0, 44.0, 55.0, 66.0, 77.0, 88.0} FloatVector := []float32{1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0, 11.0, 22.0, 33.0, 44.0, 55.0, 66.0, 77.0, 88.0}
Float16Vector := []byte{0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77,
0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77, 0x00, 0x11, 0x22, 0x33, 0x44, 0x55, 0x66, 0x77}
boolData := genFieldData(fieldName, fieldID, schemapb.DataType_Bool, BoolArray, 1) boolData := genFieldData(fieldName, fieldID, schemapb.DataType_Bool, BoolArray, 1)
int8Data := genFieldData(fieldName, fieldID, schemapb.DataType_Int8, Int8Array, 1) int8Data := genFieldData(fieldName, fieldID, schemapb.DataType_Int8, Int8Array, 1)
@ -1102,6 +1133,7 @@ func TestGetDataAndGetDataSize(t *testing.T) {
varCharData := genFieldData(fieldName, fieldID, schemapb.DataType_VarChar, VarCharArray, 1) varCharData := genFieldData(fieldName, fieldID, schemapb.DataType_VarChar, VarCharArray, 1)
binVecData := genFieldData(fieldName, fieldID, schemapb.DataType_BinaryVector, BinaryVector, Dim) binVecData := genFieldData(fieldName, fieldID, schemapb.DataType_BinaryVector, BinaryVector, Dim)
floatVecData := genFieldData(fieldName, fieldID, schemapb.DataType_FloatVector, FloatVector, Dim) floatVecData := genFieldData(fieldName, fieldID, schemapb.DataType_FloatVector, FloatVector, Dim)
float16VecData := genFieldData(fieldName, fieldID, schemapb.DataType_Float16Vector, Float16Vector, Dim)
invalidData := &schemapb.FieldData{ invalidData := &schemapb.FieldData{
Type: schemapb.DataType_None, Type: schemapb.DataType_None,
} }
@ -1125,6 +1157,7 @@ func TestGetDataAndGetDataSize(t *testing.T) {
varCharDataRes := GetData(varCharData, 0) varCharDataRes := GetData(varCharData, 0)
binVecDataRes := GetData(binVecData, 0) binVecDataRes := GetData(binVecData, 0)
floatVecDataRes := GetData(floatVecData, 0) floatVecDataRes := GetData(floatVecData, 0)
float16VecDataRes := GetData(float16VecData, 0)
invalidDataRes := GetData(invalidData, 0) invalidDataRes := GetData(invalidData, 0)
assert.Equal(t, BoolArray[0], boolDataRes) assert.Equal(t, BoolArray[0], boolDataRes)
@ -1137,6 +1170,7 @@ func TestGetDataAndGetDataSize(t *testing.T) {
assert.Equal(t, VarCharArray[0], varCharDataRes) assert.Equal(t, VarCharArray[0], varCharDataRes)
assert.ElementsMatch(t, BinaryVector[:Dim/8], binVecDataRes) assert.ElementsMatch(t, BinaryVector[:Dim/8], binVecDataRes)
assert.ElementsMatch(t, FloatVector[:Dim], floatVecDataRes) assert.ElementsMatch(t, FloatVector[:Dim], floatVecDataRes)
assert.ElementsMatch(t, Float16Vector[:2*Dim], float16VecDataRes)
assert.Nil(t, invalidDataRes) assert.Nil(t, invalidDataRes)
}) })
} }

View File

@ -125,6 +125,8 @@ func (s *TestGetVectorSuite) run() {
var vecFieldData *schemapb.FieldData var vecFieldData *schemapb.FieldData
if s.vecType == schemapb.DataType_FloatVector { if s.vecType == schemapb.DataType_FloatVector {
vecFieldData = integration.NewFloatVectorFieldData(vecFieldName, NB, dim) vecFieldData = integration.NewFloatVectorFieldData(vecFieldName, NB, dim)
} else if s.vecType == schemapb.DataType_Float16Vector {
vecFieldData = integration.NewFloat16VectorFieldData(vecFieldName, NB, dim)
} else { } else {
vecFieldData = integration.NewBinaryVectorFieldData(vecFieldName, NB, dim) vecFieldData = integration.NewBinaryVectorFieldData(vecFieldName, NB, dim)
} }
@ -235,6 +237,25 @@ func (s *TestGetVectorSuite) run() {
s.Require().ElementsMatch(expect, actual) s.Require().ElementsMatch(expect, actual)
} }
} }
} else if s.vecType == schemapb.DataType_Float16Vector {
// s.Require().Len(result.GetFieldsData()[vecFieldIndex].GetVectors().GetFloat16Vector(), nq*topk*dim*2)
// rawData := vecFieldData.GetVectors().GetFloat16Vector()
// resData := result.GetFieldsData()[vecFieldIndex].GetVectors().GetFloat16Vector()
// if s.pkType == schemapb.DataType_Int64 {
// for i, id := range result.GetIds().GetIntId().GetData() {
// expect := rawData[int(id)*dim : (int(id)+1)*dim]
// actual := resData[i*dim : (i+1)*dim]
// s.Require().ElementsMatch(expect, actual)
// }
// } else {
// for i, idStr := range result.GetIds().GetStrId().GetData() {
// id, err := strconv.Atoi(idStr)
// s.Require().NoError(err)
// expect := rawData[id*dim : (id+1)*dim]
// actual := resData[i*dim : (i+1)*dim]
// s.Require().ElementsMatch(expect, actual)
// }
// }
} else { } else {
s.Require().Len(result.GetFieldsData()[vecFieldIndex].GetVectors().GetBinaryVector(), nq*topk*dim/8) s.Require().Len(result.GetFieldsData()[vecFieldIndex].GetVectors().GetBinaryVector(), nq*topk*dim/8)
rawData := vecFieldData.GetVectors().GetBinaryVector() rawData := vecFieldData.GetVectors().GetBinaryVector()
@ -281,6 +302,17 @@ func (s *TestGetVectorSuite) TestGetVector_FLAT() {
s.run() s.run()
} }
func (s *TestGetVectorSuite) TestGetVector_Float16Vector() {
s.nq = 10
s.topK = 10
s.indexType = integration.IndexFaissIDMap
s.metricType = metric.L2
s.pkType = schemapb.DataType_Int64
s.vecType = schemapb.DataType_Float16Vector
s.searchFailed = false
s.run()
}
func (s *TestGetVectorSuite) TestGetVector_IVF_FLAT() { func (s *TestGetVectorSuite) TestGetVector_IVF_FLAT() {
s.nq = 10 s.nq = 10
s.topK = 10 s.topK = 10

View File

@ -116,6 +116,21 @@ func NewFloatVectorFieldData(fieldName string, numRows, dim int) *schemapb.Field
} }
} }
func NewFloat16VectorFieldData(fieldName string, numRows, dim int) *schemapb.FieldData {
return &schemapb.FieldData{
Type: schemapb.DataType_Float16Vector,
FieldName: fieldName,
Field: &schemapb.FieldData_Vectors{
Vectors: &schemapb.VectorField{
Dim: int64(dim),
Data: &schemapb.VectorField_Float16Vector{
Float16Vector: GenerateFloat16Vectors(numRows, dim),
},
},
},
}
}
func NewBinaryVectorFieldData(fieldName string, numRows, dim int) *schemapb.FieldData { func NewBinaryVectorFieldData(fieldName string, numRows, dim int) *schemapb.FieldData {
return &schemapb.FieldData{ return &schemapb.FieldData{
Type: schemapb.DataType_BinaryVector, Type: schemapb.DataType_BinaryVector,
@ -166,6 +181,16 @@ func GenerateBinaryVectors(numRows, dim int) []byte {
return ret return ret
} }
func GenerateFloat16Vectors(numRows, dim int) []byte {
total := numRows * dim * 2
ret := make([]byte, total)
_, err := rand.Read(ret)
if err != nil {
panic(err)
}
return ret
}
func GenerateHashKeys(numRows int) []uint32 { func GenerateHashKeys(numRows int) []uint32 {
ret := make([]uint32, 0, numRows) ret := make([]uint32, 0, numRows)
for i := 0; i < numRows; i++ { for i := 0; i < numRows; i++ {

View File

@ -181,6 +181,17 @@ func constructPlaceholderGroup(nq, dim int, vectorType schemapb.DataType) *commo
} }
values = append(values, ret) values = append(values, ret)
} }
case schemapb.DataType_Float16Vector:
placeholderType = commonpb.PlaceholderType_Float16Vector
for i := 0; i < nq; i++ {
total := dim * 2
ret := make([]byte, total)
_, err := rand.Read(ret)
if err != nil {
panic(err)
}
values = append(values, ret)
}
default: default:
panic("invalid vector data type") panic("invalid vector data type")
} }

View File

@ -25,17 +25,18 @@ import (
) )
const ( const (
BoolField = "boolField" BoolField = "boolField"
Int8Field = "int8Field" Int8Field = "int8Field"
Int16Field = "int16Field" Int16Field = "int16Field"
Int32Field = "int32Field" Int32Field = "int32Field"
Int64Field = "int64Field" Int64Field = "int64Field"
FloatField = "floatField" FloatField = "floatField"
DoubleField = "doubleField" DoubleField = "doubleField"
VarCharField = "varCharField" VarCharField = "varCharField"
JSONField = "jsonField" JSONField = "jsonField"
FloatVecField = "floatVecField" FloatVecField = "floatVecField"
BinVecField = "binVecField" BinVecField = "binVecField"
Float16VecField = "float16VecField"
) )
func ConstructSchema(collection string, dim int, autoID bool, fields ...*schemapb.FieldSchema) *schemapb.CollectionSchema { func ConstructSchema(collection string, dim int, autoID bool, fields ...*schemapb.FieldSchema) *schemapb.CollectionSchema {