diff --git a/go.mod b/go.mod index fe0e9b7070..8b1b74eb0b 100644 --- a/go.mod +++ b/go.mod @@ -20,7 +20,7 @@ require ( github.com/golang/protobuf v1.5.3 github.com/klauspost/compress v1.14.4 github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d - github.com/milvus-io/milvus-proto/go-api v0.0.0-20230517025117-8ba62a3f3a63 + github.com/milvus-io/milvus-proto/go-api v0.0.0-20230518083323-3400e837ef47 github.com/milvus-io/milvus/pkg v0.0.0-00010101000000-000000000000 github.com/minio/minio-go/v7 v7.0.17 github.com/panjf2000/ants/v2 v2.7.2 diff --git a/go.sum b/go.sum index 80a0b8eca1..b3f6d1cc61 100644 --- a/go.sum +++ b/go.sum @@ -585,6 +585,8 @@ github.com/milvus-io/milvus-proto/go-api v0.0.0-20230515081521-d963c95b041f h1:u github.com/milvus-io/milvus-proto/go-api v0.0.0-20230515081521-d963c95b041f/go.mod h1:148qnlmZ0Fdm1Fq+Mj/OW2uDoEP25g3mjh0vMGtkgmk= github.com/milvus-io/milvus-proto/go-api v0.0.0-20230517025117-8ba62a3f3a63 h1:V962mSHjOFUbuMgAXziBdbYPOCVZmN1MkqEeKpME+MA= github.com/milvus-io/milvus-proto/go-api v0.0.0-20230517025117-8ba62a3f3a63/go.mod h1:148qnlmZ0Fdm1Fq+Mj/OW2uDoEP25g3mjh0vMGtkgmk= +github.com/milvus-io/milvus-proto/go-api v0.0.0-20230518083323-3400e837ef47 h1:Dp5AAbOSTq31QLatGXamBMk/o670MkbRi8NoW17ypew= +github.com/milvus-io/milvus-proto/go-api v0.0.0-20230518083323-3400e837ef47/go.mod h1:148qnlmZ0Fdm1Fq+Mj/OW2uDoEP25g3mjh0vMGtkgmk= github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A= github.com/milvus-io/pulsar-client-go v0.6.10/go.mod h1:lQqCkgwDF8YFYjKA+zOheTk1tev2B+bKj5j7+nm8M1w= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= diff --git a/internal/core/src/pb/schema.pb.cc b/internal/core/src/pb/schema.pb.cc index 7847f8f4c1..22950333eb 100755 --- a/internal/core/src/pb/schema.pb.cc +++ b/internal/core/src/pb/schema.pb.cc @@ -231,8 +231,9 @@ PROTOBUF_ATTRIBUTE_NO_DESTROY PROTOBUF_CONSTINIT PROTOBUF_ATTRIBUTE_INIT_PRIORIT PROTOBUF_CONSTEXPR FieldData::FieldData( ::_pbi::ConstantInitialized): _impl_{ /*decltype(_impl_.field_name_)*/{&::_pbi::fixed_address_empty_string, ::_pbi::ConstantInitialized{}} - , /*decltype(_impl_.field_id_)*/int64_t{0} , /*decltype(_impl_.type_)*/0 + , /*decltype(_impl_.is_dynamic_)*/false + , /*decltype(_impl_.field_id_)*/int64_t{0} , /*decltype(_impl_.field_)*/{} , /*decltype(_impl_._cached_size_)*/{} , /*decltype(_impl_._oneof_case_)*/{}} {} @@ -431,6 +432,7 @@ const uint32_t TableStruct_schema_2eproto::offsets[] PROTOBUF_SECTION_VARIABLE(p ::_pbi::kInvalidFieldOffsetTag, ::_pbi::kInvalidFieldOffsetTag, PROTOBUF_FIELD_OFFSET(::milvus::proto::schema::FieldData, _impl_.field_id_), + PROTOBUF_FIELD_OFFSET(::milvus::proto::schema::FieldData, _impl_.is_dynamic_), PROTOBUF_FIELD_OFFSET(::milvus::proto::schema::FieldData, _impl_.field_), ~0u, // no _has_bits_ PROTOBUF_FIELD_OFFSET(::milvus::proto::schema::IDs, _internal_metadata_), @@ -470,8 +472,8 @@ static const ::_pbi::MigrationSchema schemas[] PROTOBUF_SECTION_VARIABLE(protode { 108, -1, -1, sizeof(::milvus::proto::schema::ScalarField)}, { 124, -1, -1, sizeof(::milvus::proto::schema::VectorField)}, { 134, -1, -1, sizeof(::milvus::proto::schema::FieldData)}, - { 146, -1, -1, sizeof(::milvus::proto::schema::IDs)}, - { 155, -1, -1, sizeof(::milvus::proto::schema::SearchResultData)}, + { 147, -1, -1, sizeof(::milvus::proto::schema::IDs)}, + { 156, -1, -1, sizeof(::milvus::proto::schema::SearchResultData)}, }; static const ::_pb::Message* const file_default_instances[] = { @@ -540,36 +542,37 @@ const char descriptor_table_protodef_schema_2eproto[] PROTOBUF_SECTION_VARIABLE( "JSONArrayH\000B\006\n\004data\"t\n\013VectorField\022\013\n\003di" "m\030\001 \001(\003\0227\n\014float_vector\030\002 \001(\0132\037.milvus.p" "roto.schema.FloatArrayH\000\022\027\n\rbinary_vecto" - "r\030\003 \001(\014H\000B\006\n\004data\"\321\001\n\tFieldData\022+\n\004type\030" + "r\030\003 \001(\014H\000B\006\n\004data\"\345\001\n\tFieldData\022+\n\004type\030" "\001 \001(\0162\035.milvus.proto.schema.DataType\022\022\n\n" "field_name\030\002 \001(\t\0223\n\007scalars\030\003 \001(\0132 .milv" "us.proto.schema.ScalarFieldH\000\0223\n\007vectors" "\030\004 \001(\0132 .milvus.proto.schema.VectorField" - "H\000\022\020\n\010field_id\030\005 \001(\003B\007\n\005field\"w\n\003IDs\0220\n\006" - "int_id\030\001 \001(\0132\036.milvus.proto.schema.LongA" - "rrayH\000\0222\n\006str_id\030\002 \001(\0132 .milvus.proto.sc" - "hema.StringArrayH\000B\n\n\010id_field\"\261\001\n\020Searc" - "hResultData\022\023\n\013num_queries\030\001 \001(\003\022\r\n\005top_" - "k\030\002 \001(\003\0223\n\013fields_data\030\003 \003(\0132\036.milvus.pr" - "oto.schema.FieldData\022\016\n\006scores\030\004 \003(\002\022%\n\003" - "ids\030\005 \001(\0132\030.milvus.proto.schema.IDs\022\r\n\005t" - "opks\030\006 \003(\003*\261\001\n\010DataType\022\010\n\004None\020\000\022\010\n\004Boo" - "l\020\001\022\010\n\004Int8\020\002\022\t\n\005Int16\020\003\022\t\n\005Int32\020\004\022\t\n\005I" - "nt64\020\005\022\t\n\005Float\020\n\022\n\n\006Double\020\013\022\n\n\006String\020" - "\024\022\013\n\007VarChar\020\025\022\t\n\005Array\020\026\022\010\n\004JSON\020\027\022\020\n\014B" - "inaryVector\020d\022\017\n\013FloatVector\020e*V\n\nFieldS" - "tate\022\020\n\014FieldCreated\020\000\022\021\n\rFieldCreating\020" - "\001\022\021\n\rFieldDropping\020\002\022\020\n\014FieldDropped\020\003Bf" - "\n\016io.milvus.grpcB\013SchemaProtoP\001Z1github." - "com/milvus-io/milvus-proto/go-api/schema" - "pb\240\001\001\252\002\016IO.Milvus.Grpcb\006proto3" + "H\000\022\020\n\010field_id\030\005 \001(\003\022\022\n\nis_dynamic\030\006 \001(\010" + "B\007\n\005field\"w\n\003IDs\0220\n\006int_id\030\001 \001(\0132\036.milvu" + "s.proto.schema.LongArrayH\000\0222\n\006str_id\030\002 \001" + "(\0132 .milvus.proto.schema.StringArrayH\000B\n" + "\n\010id_field\"\261\001\n\020SearchResultData\022\023\n\013num_q" + "ueries\030\001 \001(\003\022\r\n\005top_k\030\002 \001(\003\0223\n\013fields_da" + "ta\030\003 \003(\0132\036.milvus.proto.schema.FieldData" + "\022\016\n\006scores\030\004 \003(\002\022%\n\003ids\030\005 \001(\0132\030.milvus.p" + "roto.schema.IDs\022\r\n\005topks\030\006 \003(\003*\261\001\n\010DataT" + "ype\022\010\n\004None\020\000\022\010\n\004Bool\020\001\022\010\n\004Int8\020\002\022\t\n\005Int" + "16\020\003\022\t\n\005Int32\020\004\022\t\n\005Int64\020\005\022\t\n\005Float\020\n\022\n\n" + "\006Double\020\013\022\n\n\006String\020\024\022\013\n\007VarChar\020\025\022\t\n\005Ar" + "ray\020\026\022\010\n\004JSON\020\027\022\020\n\014BinaryVector\020d\022\017\n\013Flo" + "atVector\020e*V\n\nFieldState\022\020\n\014FieldCreated" + "\020\000\022\021\n\rFieldCreating\020\001\022\021\n\rFieldDropping\020\002" + "\022\020\n\014FieldDropped\020\003Bf\n\016io.milvus.grpcB\013Sc" + "hemaProtoP\001Z1github.com/milvus-io/milvus" + "-proto/go-api/schemapb\240\001\001\252\002\016IO.Milvus.Gr" + "pcb\006proto3" ; static const ::_pbi::DescriptorTable* const descriptor_table_schema_2eproto_deps[1] = { &::descriptor_table_common_2eproto, }; static ::_pbi::once_flag descriptor_table_schema_2eproto_once; const ::_pbi::DescriptorTable descriptor_table_schema_2eproto = { - false, false, 2710, descriptor_table_protodef_schema_2eproto, + false, false, 2730, descriptor_table_protodef_schema_2eproto, "schema.proto", &descriptor_table_schema_2eproto_once, descriptor_table_schema_2eproto_deps, 1, 17, schemas, file_default_instances, TableStruct_schema_2eproto::offsets, @@ -4762,8 +4765,9 @@ FieldData::FieldData(const FieldData& from) FieldData* const _this = this; (void)_this; new (&_impl_) Impl_{ decltype(_impl_.field_name_){} - , decltype(_impl_.field_id_){} , decltype(_impl_.type_){} + , decltype(_impl_.is_dynamic_){} + , decltype(_impl_.field_id_){} , decltype(_impl_.field_){} , /*decltype(_impl_._cached_size_)*/{} , /*decltype(_impl_._oneof_case_)*/{}}; @@ -4777,9 +4781,9 @@ FieldData::FieldData(const FieldData& from) _this->_impl_.field_name_.Set(from._internal_field_name(), _this->GetArenaForAllocation()); } - ::memcpy(&_impl_.field_id_, &from._impl_.field_id_, - static_cast(reinterpret_cast(&_impl_.type_) - - reinterpret_cast(&_impl_.field_id_)) + sizeof(_impl_.type_)); + ::memcpy(&_impl_.type_, &from._impl_.type_, + static_cast(reinterpret_cast(&_impl_.field_id_) - + reinterpret_cast(&_impl_.type_)) + sizeof(_impl_.field_id_)); clear_has_field(); switch (from.field_case()) { case kScalars: { @@ -4805,8 +4809,9 @@ inline void FieldData::SharedCtor( (void)is_message_owned; new (&_impl_) Impl_{ decltype(_impl_.field_name_){} - , decltype(_impl_.field_id_){int64_t{0}} , decltype(_impl_.type_){0} + , decltype(_impl_.is_dynamic_){false} + , decltype(_impl_.field_id_){int64_t{0}} , decltype(_impl_.field_){} , /*decltype(_impl_._cached_size_)*/{} , /*decltype(_impl_._oneof_case_)*/{} @@ -4869,9 +4874,9 @@ void FieldData::Clear() { (void) cached_has_bits; _impl_.field_name_.ClearToEmpty(); - ::memset(&_impl_.field_id_, 0, static_cast( - reinterpret_cast(&_impl_.type_) - - reinterpret_cast(&_impl_.field_id_)) + sizeof(_impl_.type_)); + ::memset(&_impl_.type_, 0, static_cast( + reinterpret_cast(&_impl_.field_id_) - + reinterpret_cast(&_impl_.type_)) + sizeof(_impl_.field_id_)); clear_field(); _internal_metadata_.Clear<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(); } @@ -4925,6 +4930,14 @@ const char* FieldData::_InternalParse(const char* ptr, ::_pbi::ParseContext* ctx } else goto handle_unusual; continue; + // bool is_dynamic = 6; + case 6: + if (PROTOBUF_PREDICT_TRUE(static_cast(tag) == 48)) { + _impl_.is_dynamic_ = ::PROTOBUF_NAMESPACE_ID::internal::ReadVarint64(&ptr); + CHK_(ptr); + } else + goto handle_unusual; + continue; default: goto handle_unusual; } // switch @@ -4991,6 +5004,12 @@ uint8_t* FieldData::_InternalSerialize( target = ::_pbi::WireFormatLite::WriteInt64ToArray(5, this->_internal_field_id(), target); } + // bool is_dynamic = 6; + if (this->_internal_is_dynamic() != 0) { + target = stream->EnsureSpace(target); + target = ::_pbi::WireFormatLite::WriteBoolToArray(6, this->_internal_is_dynamic(), target); + } + if (PROTOBUF_PREDICT_FALSE(_internal_metadata_.have_unknown_fields())) { target = ::_pbi::WireFormat::InternalSerializeUnknownFieldsToArray( _internal_metadata_.unknown_fields<::PROTOBUF_NAMESPACE_ID::UnknownFieldSet>(::PROTOBUF_NAMESPACE_ID::UnknownFieldSet::default_instance), target, stream); @@ -5014,17 +5033,22 @@ size_t FieldData::ByteSizeLong() const { this->_internal_field_name()); } - // int64 field_id = 5; - if (this->_internal_field_id() != 0) { - total_size += ::_pbi::WireFormatLite::Int64SizePlusOne(this->_internal_field_id()); - } - // .milvus.proto.schema.DataType type = 1; if (this->_internal_type() != 0) { total_size += 1 + ::_pbi::WireFormatLite::EnumSize(this->_internal_type()); } + // bool is_dynamic = 6; + if (this->_internal_is_dynamic() != 0) { + total_size += 1 + 1; + } + + // int64 field_id = 5; + if (this->_internal_field_id() != 0) { + total_size += ::_pbi::WireFormatLite::Int64SizePlusOne(this->_internal_field_id()); + } + switch (field_case()) { // .milvus.proto.schema.ScalarField scalars = 3; case kScalars: { @@ -5065,12 +5089,15 @@ void FieldData::MergeImpl(::PROTOBUF_NAMESPACE_ID::Message& to_msg, const ::PROT if (!from._internal_field_name().empty()) { _this->_internal_set_field_name(from._internal_field_name()); } - if (from._internal_field_id() != 0) { - _this->_internal_set_field_id(from._internal_field_id()); - } if (from._internal_type() != 0) { _this->_internal_set_type(from._internal_type()); } + if (from._internal_is_dynamic() != 0) { + _this->_internal_set_is_dynamic(from._internal_is_dynamic()); + } + if (from._internal_field_id() != 0) { + _this->_internal_set_field_id(from._internal_field_id()); + } switch (from.field_case()) { case kScalars: { _this->_internal_mutable_scalars()->::milvus::proto::schema::ScalarField::MergeFrom( @@ -5110,11 +5137,11 @@ void FieldData::InternalSwap(FieldData* other) { &other->_impl_.field_name_, rhs_arena ); ::PROTOBUF_NAMESPACE_ID::internal::memswap< - PROTOBUF_FIELD_OFFSET(FieldData, _impl_.type_) - + sizeof(FieldData::_impl_.type_) - - PROTOBUF_FIELD_OFFSET(FieldData, _impl_.field_id_)>( - reinterpret_cast(&_impl_.field_id_), - reinterpret_cast(&other->_impl_.field_id_)); + PROTOBUF_FIELD_OFFSET(FieldData, _impl_.field_id_) + + sizeof(FieldData::_impl_.field_id_) + - PROTOBUF_FIELD_OFFSET(FieldData, _impl_.type_)>( + reinterpret_cast(&_impl_.type_), + reinterpret_cast(&other->_impl_.type_)); swap(_impl_.field_, other->_impl_.field_); swap(_impl_._oneof_case_[0], other->_impl_._oneof_case_[0]); } diff --git a/internal/core/src/pb/schema.pb.h b/internal/core/src/pb/schema.pb.h index 97059578e5..8ddfeac019 100755 --- a/internal/core/src/pb/schema.pb.h +++ b/internal/core/src/pb/schema.pb.h @@ -3150,8 +3150,9 @@ class FieldData final : enum : int { kFieldNameFieldNumber = 2, - kFieldIdFieldNumber = 5, kTypeFieldNumber = 1, + kIsDynamicFieldNumber = 6, + kFieldIdFieldNumber = 5, kScalarsFieldNumber = 3, kVectorsFieldNumber = 4, }; @@ -3169,15 +3170,6 @@ class FieldData final : std::string* _internal_mutable_field_name(); public: - // int64 field_id = 5; - void clear_field_id(); - int64_t field_id() const; - void set_field_id(int64_t value); - private: - int64_t _internal_field_id() const; - void _internal_set_field_id(int64_t value); - public: - // .milvus.proto.schema.DataType type = 1; void clear_type(); ::milvus::proto::schema::DataType type() const; @@ -3187,6 +3179,24 @@ class FieldData final : void _internal_set_type(::milvus::proto::schema::DataType value); public: + // bool is_dynamic = 6; + void clear_is_dynamic(); + bool is_dynamic() const; + void set_is_dynamic(bool value); + private: + bool _internal_is_dynamic() const; + void _internal_set_is_dynamic(bool value); + public: + + // int64 field_id = 5; + void clear_field_id(); + int64_t field_id() const; + void set_field_id(int64_t value); + private: + int64_t _internal_field_id() const; + void _internal_set_field_id(int64_t value); + public: + // .milvus.proto.schema.ScalarField scalars = 3; bool has_scalars() const; private: @@ -3239,8 +3249,9 @@ class FieldData final : typedef void DestructorSkippable_; struct Impl_ { ::PROTOBUF_NAMESPACE_ID::internal::ArenaStringPtr field_name_; - int64_t field_id_; int type_; + bool is_dynamic_; + int64_t field_id_; union FieldUnion { constexpr FieldUnion() : _constinit_{} {} ::PROTOBUF_NAMESPACE_ID::internal::ConstantInitialized _constinit_; @@ -6335,6 +6346,26 @@ inline void FieldData::set_field_id(int64_t value) { // @@protoc_insertion_point(field_set:milvus.proto.schema.FieldData.field_id) } +// bool is_dynamic = 6; +inline void FieldData::clear_is_dynamic() { + _impl_.is_dynamic_ = false; +} +inline bool FieldData::_internal_is_dynamic() const { + return _impl_.is_dynamic_; +} +inline bool FieldData::is_dynamic() const { + // @@protoc_insertion_point(field_get:milvus.proto.schema.FieldData.is_dynamic) + return _internal_is_dynamic(); +} +inline void FieldData::_internal_set_is_dynamic(bool value) { + + _impl_.is_dynamic_ = value; +} +inline void FieldData::set_is_dynamic(bool value) { + _internal_set_is_dynamic(value); + // @@protoc_insertion_point(field_set:milvus.proto.schema.FieldData.is_dynamic) +} + inline bool FieldData::has_field() const { return field_case() != FIELD_NOT_SET; } diff --git a/internal/proxy/default_limit_reducer.go b/internal/proxy/default_limit_reducer.go index 278c0e0f8a..845453bb80 100644 --- a/internal/proxy/default_limit_reducer.go +++ b/internal/proxy/default_limit_reducer.go @@ -70,6 +70,7 @@ func (r *defaultLimitReducer) afterReduce(result *milvuspb.QueryResults) error { result.FieldsData[i].FieldName = field.Name result.FieldsData[i].FieldId = field.FieldID result.FieldsData[i].Type = field.DataType + result.FieldsData[i].IsDynamic = field.IsDynamic } } } diff --git a/internal/proxy/task_search.go b/internal/proxy/task_search.go index 55e8b40a1a..836b639ba8 100644 --- a/internal/proxy/task_search.go +++ b/internal/proxy/task_search.go @@ -612,6 +612,7 @@ func (t *searchTask) fillInFieldInfo() { t.result.Results.FieldsData[i].FieldName = field.Name t.result.Results.FieldsData[i].FieldId = field.FieldID t.result.Results.FieldsData[i].Type = field.DataType + t.result.Results.FieldsData[i].IsDynamic = field.IsDynamic } } } diff --git a/pkg/go.mod b/pkg/go.mod index 816446adc9..5b82ceeb24 100644 --- a/pkg/go.mod +++ b/pkg/go.mod @@ -12,7 +12,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/klauspost/compress v1.14.4 github.com/lingdor/stackerror v0.0.0-20191119040541-976d8885ed76 - github.com/milvus-io/milvus-proto/go-api v0.0.0-20230517025117-8ba62a3f3a63 + github.com/milvus-io/milvus-proto/go-api v0.0.0-20230518083323-3400e837ef47 github.com/panjf2000/ants/v2 v2.4.8 github.com/prometheus/client_golang v1.11.1 github.com/samber/lo v1.27.0 diff --git a/pkg/go.sum b/pkg/go.sum index ff2fa6396a..c333985622 100644 --- a/pkg/go.sum +++ b/pkg/go.sum @@ -471,6 +471,8 @@ github.com/milvus-io/milvus-proto/go-api v0.0.0-20230515081521-d963c95b041f h1:u github.com/milvus-io/milvus-proto/go-api v0.0.0-20230515081521-d963c95b041f/go.mod h1:148qnlmZ0Fdm1Fq+Mj/OW2uDoEP25g3mjh0vMGtkgmk= github.com/milvus-io/milvus-proto/go-api v0.0.0-20230517025117-8ba62a3f3a63 h1:V962mSHjOFUbuMgAXziBdbYPOCVZmN1MkqEeKpME+MA= github.com/milvus-io/milvus-proto/go-api v0.0.0-20230517025117-8ba62a3f3a63/go.mod h1:148qnlmZ0Fdm1Fq+Mj/OW2uDoEP25g3mjh0vMGtkgmk= +github.com/milvus-io/milvus-proto/go-api v0.0.0-20230518083323-3400e837ef47 h1:Dp5AAbOSTq31QLatGXamBMk/o670MkbRi8NoW17ypew= +github.com/milvus-io/milvus-proto/go-api v0.0.0-20230518083323-3400e837ef47/go.mod h1:148qnlmZ0Fdm1Fq+Mj/OW2uDoEP25g3mjh0vMGtkgmk= github.com/milvus-io/pulsar-client-go v0.6.10 h1:eqpJjU+/QX0iIhEo3nhOqMNXL+TyInAs1IAHZCrCM/A= github.com/milvus-io/pulsar-client-go v0.6.10/go.mod h1:lQqCkgwDF8YFYjKA+zOheTk1tev2B+bKj5j7+nm8M1w= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=