From e45cb6c644a0ce690f734a689f7e2331eb4dc251 Mon Sep 17 00:00:00 2001 From: dragondriver Date: Wed, 18 Aug 2021 10:12:10 +0800 Subject: [PATCH] Expose system topology information by proxy (#7137) Signed-off-by: dragondriver --- internal/core/src/pb/milvus.pb.cc | 17 +- internal/distributed/proxy/service.go | 4 + internal/proto/milvus.proto | 3 + internal/proto/milvuspb/milvus.pb.go | 414 ++++++++++-------- internal/proxy/error.go | 19 + internal/proxy/error_test.go | 41 ++ internal/proxy/impl.go | 69 +++ internal/proxy/metrics_info.go | 128 ++++++ internal/querycoord/metrics_info.go | 67 ++- internal/querynode/metrics_info.go | 8 +- internal/util/metricsinfo/metrics_info.go | 57 +++ .../util/metricsinfo/metrics_info_test.go | 69 +++ internal/util/metricsinfo/topology.go | 85 ++-- internal/util/metricsinfo/topology_test.go | 85 ++-- 14 files changed, 766 insertions(+), 300 deletions(-) create mode 100644 internal/proxy/error_test.go create mode 100644 internal/proxy/metrics_info.go create mode 100644 internal/util/metricsinfo/metrics_info.go create mode 100644 internal/util/metricsinfo/metrics_info_test.go diff --git a/internal/core/src/pb/milvus.pb.cc b/internal/core/src/pb/milvus.pb.cc index 9cd7cd3c85..606825ea1d 100644 --- a/internal/core/src/pb/milvus.pb.cc +++ b/internal/core/src/pb/milvus.pb.cc @@ -2166,7 +2166,7 @@ const char descriptor_table_protodef_milvus_2eproto[] PROTOBUF_SECTION_VARIABLE( "tatus\022\020\n\010response\030\002 \001(\t\022\026\n\016component_nam" "e\030\003 \001(\t*!\n\010ShowType\022\007\n\003All\020\000\022\014\n\010InMemory" "\020\001*>\n\017PlaceholderType\022\010\n\004None\020\000\022\020\n\014Binar" - "yVector\020d\022\017\n\013FloatVector\020e2\240\027\n\rMilvusSer" + "yVector\020d\022\017\n\013FloatVector\020e2\201\030\n\rMilvusSer" "vice\022_\n\020CreateCollection\022,.milvus.proto." "milvus.CreateCollectionRequest\032\033.milvus." "proto.common.Status\"\000\022[\n\016DropCollection\022" @@ -2241,11 +2241,14 @@ const char descriptor_table_protodef_milvus_2eproto[] PROTOBUF_SECTION_VARIABLE( "milvus.DummyResponse\"\000\022e\n\014RegisterLink\022(" ".milvus.proto.milvus.RegisterLinkRequest" "\032).milvus.proto.milvus.RegisterLinkRespo" - "nse\"\0002u\n\014ProxyService\022e\n\014RegisterLink\022(." - "milvus.proto.milvus.RegisterLinkRequest\032" - ").milvus.proto.milvus.RegisterLinkRespon" - "se\"\000B5Z3github.com/milvus-io/milvus/inte" - "rnal/proto/milvuspbb\006proto3" + "nse\"\000\022_\n\nGetMetrics\022&.milvus.proto.milvu" + "s.GetMetricsRequest\032\'.milvus.proto.milvu" + "s.GetMetricsResponse\"\0002u\n\014ProxyService\022e" + "\n\014RegisterLink\022(.milvus.proto.milvus.Reg" + "isterLinkRequest\032).milvus.proto.milvus.R" + "egisterLinkResponse\"\000B5Z3github.com/milv" + "us-io/milvus/internal/proto/milvuspbb\006pr" + "oto3" ; static const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable*const descriptor_table_milvus_2eproto_deps[2] = { &::descriptor_table_common_2eproto, @@ -2319,7 +2322,7 @@ static ::PROTOBUF_NAMESPACE_ID::internal::SCCInfoBase*const descriptor_table_mil static ::PROTOBUF_NAMESPACE_ID::internal::once_flag descriptor_table_milvus_2eproto_once; static bool descriptor_table_milvus_2eproto_initialized = false; const ::PROTOBUF_NAMESPACE_ID::internal::DescriptorTable descriptor_table_milvus_2eproto = { - &descriptor_table_milvus_2eproto_initialized, descriptor_table_protodef_milvus_2eproto, "milvus.proto", 12027, + &descriptor_table_milvus_2eproto_initialized, descriptor_table_protodef_milvus_2eproto, "milvus.proto", 12124, &descriptor_table_milvus_2eproto_once, descriptor_table_milvus_2eproto_sccs, descriptor_table_milvus_2eproto_deps, 63, 2, schemas, file_default_instances, TableStruct_milvus_2eproto::offsets, file_level_metadata_milvus_2eproto, 63, file_level_enum_descriptors_milvus_2eproto, file_level_service_descriptors_milvus_2eproto, diff --git a/internal/distributed/proxy/service.go b/internal/distributed/proxy/service.go index 1edb8b9635..2cb0af6dec 100644 --- a/internal/distributed/proxy/service.go +++ b/internal/distributed/proxy/service.go @@ -406,3 +406,7 @@ func (s *Server) Dummy(ctx context.Context, request *milvuspb.DummyRequest) (*mi func (s *Server) RegisterLink(ctx context.Context, request *milvuspb.RegisterLinkRequest) (*milvuspb.RegisterLinkResponse, error) { return s.proxy.RegisterLink(ctx, request) } + +func (s *Server) GetMetrics(ctx context.Context, request *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error) { + return s.proxy.GetMetrics(ctx, request) +} diff --git a/internal/proto/milvus.proto b/internal/proto/milvus.proto index 3b68e5003b..965fcb42e1 100644 --- a/internal/proto/milvus.proto +++ b/internal/proto/milvus.proto @@ -43,6 +43,9 @@ service MilvusService { // TODO: remove rpc RegisterLink(RegisterLinkRequest) returns (RegisterLinkResponse) {} + + // https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy + rpc GetMetrics(GetMetricsRequest) returns (GetMetricsResponse) {} } message CreateCollectionRequest { diff --git a/internal/proto/milvuspb/milvus.pb.go b/internal/proto/milvuspb/milvus.pb.go index fba7c4d659..dc627b3d31 100644 --- a/internal/proto/milvuspb/milvus.pb.go +++ b/internal/proto/milvuspb/milvus.pb.go @@ -4007,193 +4007,195 @@ func init() { func init() { proto.RegisterFile("milvus.proto", fileDescriptor_02345ba45cc0e303) } var fileDescriptor_02345ba45cc0e303 = []byte{ - // 2974 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x1a, 0x4d, 0x6f, 0x24, 0x47, - 0xd5, 0x3d, 0xe3, 0xf9, 0x7a, 0xd3, 0x63, 0xcf, 0x96, 0x3f, 0x76, 0x32, 0xd9, 0xcd, 0xda, 0x1d, - 0x96, 0x78, 0x77, 0x93, 0xdd, 0xac, 0x37, 0x21, 0x21, 0x01, 0x92, 0xdd, 0x35, 0xd9, 0xb5, 0xb2, - 0x1b, 0x9c, 0x76, 0x12, 0x29, 0x44, 0x51, 0xab, 0x3d, 0x5d, 0x1e, 0xb7, 0xdc, 0xd3, 0x3d, 0x74, - 0xd5, 0xac, 0x77, 0x72, 0x42, 0x0a, 0x20, 0xa1, 0x84, 0x44, 0x08, 0x04, 0xe2, 0x86, 0x80, 0x1c, - 0xb8, 0x11, 0x40, 0x02, 0x71, 0xe2, 0x90, 0x03, 0x07, 0x24, 0x3e, 0x7e, 0x01, 0x17, 0x8e, 0xf9, - 0x07, 0x48, 0xa0, 0xfa, 0xe8, 0x9e, 0xee, 0x71, 0xf5, 0x78, 0xbc, 0x93, 0x60, 0xfb, 0xd6, 0xfd, - 0xea, 0xbd, 0xaa, 0x57, 0xaf, 0x5e, 0xbd, 0x57, 0xef, 0x03, 0xf4, 0x8e, 0xeb, 0xdd, 0xeb, 0x91, - 0xcb, 0xdd, 0x30, 0xa0, 0x01, 0x9a, 0x4b, 0xfe, 0x5d, 0x16, 0x3f, 0x4d, 0xbd, 0x15, 0x74, 0x3a, - 0x81, 0x2f, 0x80, 0x4d, 0x9d, 0xb4, 0x76, 0x70, 0xc7, 0x16, 0x7f, 0xc6, 0x27, 0x1a, 0x9c, 0xbe, - 0x19, 0x62, 0x9b, 0xe2, 0x9b, 0x81, 0xe7, 0xe1, 0x16, 0x75, 0x03, 0xdf, 0xc4, 0xdf, 0xea, 0x61, - 0x42, 0xd1, 0x93, 0x30, 0xbd, 0x65, 0x13, 0xdc, 0xd0, 0x96, 0xb4, 0x95, 0xea, 0xea, 0x99, 0xcb, - 0xa9, 0xb9, 0xe5, 0x9c, 0x77, 0x49, 0xfb, 0x86, 0x4d, 0xb0, 0xc9, 0x31, 0xd1, 0x69, 0x28, 0x39, - 0x5b, 0x96, 0x6f, 0x77, 0x70, 0x23, 0xb7, 0xa4, 0xad, 0x54, 0xcc, 0xa2, 0xb3, 0xf5, 0x8a, 0xdd, - 0xc1, 0xe8, 0x31, 0x98, 0x6d, 0xc5, 0xf3, 0x0b, 0x84, 0x3c, 0x47, 0x98, 0x19, 0x80, 0x39, 0xe2, - 0x22, 0x14, 0x05, 0x7f, 0x8d, 0xe9, 0x25, 0x6d, 0x45, 0x37, 0xe5, 0x1f, 0x3a, 0x0b, 0x40, 0x76, - 0xec, 0xd0, 0x21, 0x96, 0xdf, 0xeb, 0x34, 0x0a, 0x4b, 0xda, 0x4a, 0xc1, 0xac, 0x08, 0xc8, 0x2b, - 0xbd, 0x8e, 0xf1, 0x9e, 0x06, 0x0b, 0x6b, 0x61, 0xd0, 0x3d, 0x16, 0x9b, 0x30, 0x7e, 0xad, 0xc1, - 0xfc, 0x6d, 0x9b, 0x1c, 0x0f, 0x89, 0x9e, 0x05, 0xa0, 0x6e, 0x07, 0x5b, 0x84, 0xda, 0x9d, 0x2e, - 0x97, 0xea, 0xb4, 0x59, 0x61, 0x90, 0x4d, 0x06, 0x30, 0xde, 0x04, 0xfd, 0x46, 0x10, 0x78, 0x26, - 0x26, 0xdd, 0xc0, 0x27, 0x18, 0x5d, 0x83, 0x22, 0xa1, 0x36, 0xed, 0x11, 0xc9, 0xe4, 0xc3, 0x4a, - 0x26, 0x37, 0x39, 0x8a, 0x29, 0x51, 0xd1, 0x3c, 0x14, 0xee, 0xd9, 0x5e, 0x4f, 0xf0, 0x58, 0x36, - 0xc5, 0x8f, 0xf1, 0x16, 0xcc, 0x6c, 0xd2, 0xd0, 0xf5, 0xdb, 0x9f, 0xe1, 0xe4, 0x95, 0x68, 0xf2, - 0x7f, 0x6a, 0xf0, 0xd0, 0x1a, 0x26, 0xad, 0xd0, 0xdd, 0x3a, 0x26, 0xaa, 0x6b, 0x80, 0x3e, 0x80, - 0xac, 0xaf, 0x71, 0x51, 0xe7, 0xcd, 0x14, 0x6c, 0xe8, 0x30, 0x0a, 0xc3, 0x87, 0xf1, 0xdf, 0x1c, - 0x34, 0x55, 0x9b, 0x9a, 0x44, 0x7c, 0x5f, 0x8d, 0x6f, 0x54, 0x8e, 0x13, 0x9d, 0x4f, 0x13, 0x49, - 0x6b, 0x30, 0x58, 0x6d, 0x93, 0x03, 0xe2, 0x8b, 0x37, 0xbc, 0xab, 0xbc, 0x62, 0x57, 0xab, 0xb0, - 0x70, 0xcf, 0x0d, 0x69, 0xcf, 0xf6, 0xac, 0xd6, 0x8e, 0xed, 0xfb, 0xd8, 0xe3, 0x72, 0x22, 0x8d, - 0xe9, 0xa5, 0xfc, 0x4a, 0xc5, 0x9c, 0x93, 0x83, 0x37, 0xc5, 0x18, 0x13, 0x16, 0x41, 0x4f, 0xc1, - 0x62, 0x77, 0xa7, 0x4f, 0xdc, 0xd6, 0x3e, 0xa2, 0x02, 0x27, 0x9a, 0x8f, 0x46, 0x53, 0x54, 0x97, - 0xe0, 0x54, 0x8b, 0x5b, 0x2b, 0xc7, 0x62, 0x52, 0x13, 0x62, 0x2c, 0x72, 0x31, 0xd6, 0xe5, 0xc0, - 0x6b, 0x11, 0x9c, 0xb1, 0x15, 0x21, 0xf7, 0x68, 0x2b, 0x41, 0x50, 0xe2, 0x04, 0x73, 0x72, 0xf0, - 0x75, 0xda, 0x8a, 0x69, 0xb8, 0x21, 0xb9, 0x13, 0xd8, 0xce, 0xf1, 0x30, 0x24, 0x1f, 0x68, 0xd0, - 0x30, 0xb1, 0x87, 0x6d, 0x72, 0x3c, 0x74, 0xdc, 0xf8, 0xb1, 0x06, 0x8f, 0xdc, 0xc2, 0x34, 0xa1, - 0x2d, 0xd4, 0xa6, 0x2e, 0xa1, 0x6e, 0x8b, 0x1c, 0x25, 0x5b, 0x1f, 0x6a, 0x70, 0x2e, 0x93, 0xad, - 0x49, 0x2e, 0xcf, 0x33, 0x50, 0x60, 0x5f, 0xa4, 0x91, 0x5b, 0xca, 0xaf, 0x54, 0x57, 0x97, 0x95, - 0x34, 0x2f, 0xe3, 0xfe, 0x1b, 0xcc, 0x26, 0x6d, 0xd8, 0x6e, 0x68, 0x0a, 0x7c, 0xe3, 0x5f, 0x1a, - 0x2c, 0x6e, 0xee, 0x04, 0x7b, 0x03, 0x96, 0x3e, 0x0f, 0x01, 0xa5, 0xcd, 0x49, 0x7e, 0xc8, 0x9c, - 0xa0, 0xab, 0x30, 0x4d, 0xfb, 0x5d, 0xcc, 0x2d, 0xd1, 0xcc, 0xea, 0xd9, 0xcb, 0x8a, 0xc7, 0xc1, - 0x65, 0xc6, 0xe4, 0x6b, 0xfd, 0x2e, 0x36, 0x39, 0x2a, 0xba, 0x00, 0xf5, 0x21, 0x91, 0x47, 0x17, - 0x72, 0x36, 0x2d, 0x73, 0x62, 0xfc, 0x31, 0x07, 0xa7, 0xf7, 0x6d, 0x71, 0x12, 0x61, 0xab, 0xd6, - 0xce, 0x29, 0xd7, 0x46, 0xe7, 0x21, 0xa1, 0x02, 0x96, 0xeb, 0x90, 0x46, 0x7e, 0x29, 0xbf, 0x92, - 0x37, 0x6b, 0x09, 0xbb, 0xe4, 0x10, 0xf4, 0x04, 0xa0, 0x7d, 0xe6, 0x42, 0x58, 0xa5, 0x69, 0xf3, - 0xd4, 0xb0, 0xbd, 0xe0, 0x36, 0x49, 0x69, 0x30, 0x84, 0x08, 0xa6, 0xcd, 0x79, 0x85, 0xc5, 0x20, - 0xe8, 0x2a, 0xcc, 0xbb, 0xfe, 0x5d, 0xdc, 0x09, 0xc2, 0xbe, 0xd5, 0xc5, 0x61, 0x0b, 0xfb, 0xd4, - 0x6e, 0x63, 0xd2, 0x28, 0x72, 0x8e, 0xe6, 0xa2, 0xb1, 0x8d, 0xc1, 0x90, 0xf1, 0x3b, 0x0d, 0x16, - 0xc5, 0xab, 0x6b, 0xc3, 0x0e, 0xa9, 0x7b, 0xd4, 0x9e, 0xeb, 0x3c, 0xcc, 0x74, 0x23, 0x3e, 0x04, - 0xde, 0x34, 0xc7, 0xab, 0xc5, 0x50, 0x7e, 0xcb, 0x3e, 0xd6, 0x60, 0x9e, 0x3d, 0xb2, 0x4e, 0x12, - 0xcf, 0xbf, 0xd1, 0x60, 0xee, 0xb6, 0x4d, 0x4e, 0x12, 0xcb, 0xbf, 0x97, 0x2e, 0x28, 0xe6, 0xf9, - 0x28, 0x4d, 0x2b, 0x43, 0x4c, 0x33, 0x1d, 0x79, 0xf5, 0x99, 0x14, 0xd7, 0xc4, 0xf8, 0xc3, 0xc0, - 0x57, 0x9d, 0x30, 0xce, 0xff, 0xa4, 0xc1, 0xd9, 0x5b, 0x98, 0xc6, 0x5c, 0x1f, 0x0b, 0x9f, 0x36, - 0xae, 0xb6, 0x7c, 0x20, 0x3c, 0xb2, 0x92, 0xf9, 0x23, 0xf1, 0x7c, 0xef, 0xe5, 0x60, 0x81, 0xb9, - 0x85, 0xe3, 0xa1, 0x04, 0xe3, 0x3c, 0xca, 0x15, 0x8a, 0x52, 0x50, 0x29, 0x4a, 0xec, 0x4f, 0x8b, - 0x63, 0xfb, 0x53, 0xe3, 0xb7, 0x39, 0xf1, 0x0e, 0x48, 0x4a, 0x63, 0x92, 0x63, 0x51, 0xf0, 0x9a, - 0x53, 0xf2, 0x6a, 0x80, 0x1e, 0x43, 0xd6, 0xd7, 0x22, 0xff, 0x98, 0x82, 0x1d, 0x5b, 0xf7, 0xf8, - 0xbe, 0x06, 0x8b, 0x51, 0x18, 0xb4, 0x89, 0xdb, 0x1d, 0xec, 0xd3, 0x07, 0xd7, 0xa1, 0x61, 0x0d, - 0xc8, 0x29, 0x34, 0xe0, 0x0c, 0x54, 0x88, 0x58, 0x27, 0x8e, 0x70, 0x06, 0x00, 0xe3, 0x23, 0x0d, - 0x4e, 0xef, 0x63, 0x67, 0x92, 0x43, 0x6c, 0x40, 0xc9, 0xf5, 0x1d, 0x7c, 0x3f, 0xe6, 0x26, 0xfa, - 0x65, 0x23, 0x5b, 0x3d, 0xd7, 0x73, 0x62, 0x36, 0xa2, 0x5f, 0xb4, 0x0c, 0x3a, 0xf6, 0xed, 0x2d, - 0x0f, 0x5b, 0x1c, 0x97, 0x2b, 0x72, 0xd9, 0xac, 0x0a, 0xd8, 0x3a, 0x03, 0x19, 0x3f, 0xd0, 0x60, - 0x8e, 0xe9, 0x9a, 0xe4, 0x91, 0x7c, 0xbe, 0x32, 0x5b, 0x82, 0x6a, 0x42, 0x99, 0x24, 0xbb, 0x49, - 0x90, 0xb1, 0x0b, 0xf3, 0x69, 0x76, 0x26, 0x91, 0xd9, 0x23, 0x00, 0xf1, 0x89, 0x08, 0x9d, 0xcf, - 0x9b, 0x09, 0x88, 0xf1, 0xa9, 0x06, 0x48, 0x3c, 0xa9, 0xb8, 0x30, 0x8e, 0x38, 0xe3, 0xb2, 0xed, - 0x62, 0xcf, 0x49, 0x5a, 0xed, 0x0a, 0x87, 0xf0, 0xe1, 0x35, 0xd0, 0xf1, 0x7d, 0x1a, 0xda, 0x56, - 0xd7, 0x0e, 0xed, 0x8e, 0xb8, 0x3c, 0x63, 0x19, 0xd8, 0x2a, 0x27, 0xdb, 0xe0, 0x54, 0xc6, 0x5f, - 0xd8, 0x63, 0x4c, 0x2a, 0xe5, 0x71, 0xdf, 0xf1, 0x59, 0x00, 0xae, 0xb4, 0x62, 0xb8, 0x20, 0x86, - 0x39, 0x84, 0xbb, 0xb0, 0x8f, 0x34, 0xa8, 0xf3, 0x2d, 0x88, 0xfd, 0x74, 0xd9, 0xb4, 0x43, 0x34, - 0xda, 0x10, 0xcd, 0x88, 0x2b, 0xf4, 0x65, 0x28, 0x4a, 0xc1, 0xe6, 0xc7, 0x15, 0xac, 0x24, 0x38, - 0x60, 0x1b, 0xc6, 0x2f, 0x34, 0x58, 0x18, 0x12, 0xf9, 0x24, 0x1a, 0xfd, 0x1a, 0x20, 0xb1, 0x43, - 0x67, 0xb0, 0xed, 0xc8, 0xdd, 0x9e, 0x57, 0xfa, 0x96, 0x61, 0x21, 0x99, 0xa7, 0xdc, 0x21, 0x08, - 0x31, 0xfe, 0xae, 0xc1, 0x99, 0x5b, 0x98, 0x72, 0xd4, 0x1b, 0xcc, 0x76, 0x6c, 0x84, 0x41, 0x3b, - 0xc4, 0x84, 0x9c, 0x5c, 0xfd, 0xf8, 0x89, 0x78, 0x9f, 0xa9, 0xb6, 0x34, 0x89, 0xfc, 0x97, 0x41, - 0xe7, 0x6b, 0x60, 0xc7, 0x0a, 0x83, 0x3d, 0x22, 0xf5, 0xa8, 0x2a, 0x61, 0x66, 0xb0, 0xc7, 0x15, - 0x82, 0x06, 0xd4, 0xf6, 0x04, 0x82, 0x74, 0x0c, 0x1c, 0xc2, 0x86, 0xf9, 0x1d, 0x8c, 0x18, 0x63, - 0x93, 0xe3, 0x93, 0x2b, 0xe3, 0x5f, 0x69, 0xb0, 0x30, 0xb4, 0x95, 0x49, 0x64, 0xfb, 0xb4, 0x78, - 0x3d, 0x8a, 0xcd, 0xcc, 0xac, 0x9e, 0x53, 0xd2, 0x24, 0x16, 0x13, 0xd8, 0xe8, 0x1c, 0x54, 0xb7, - 0x6d, 0xd7, 0xb3, 0x42, 0x6c, 0x93, 0xc0, 0x97, 0x1b, 0x05, 0x06, 0x32, 0x39, 0xc4, 0xf8, 0x44, - 0x83, 0x3a, 0x0b, 0x41, 0x4f, 0xb8, 0xc5, 0xfb, 0x65, 0x0e, 0x6a, 0xeb, 0x3e, 0xc1, 0x21, 0x3d, - 0xfe, 0x11, 0x06, 0x7a, 0x01, 0xaa, 0x7c, 0x63, 0xc4, 0x72, 0x6c, 0x6a, 0x4b, 0x77, 0xf5, 0x88, - 0x32, 0x8b, 0xfc, 0x12, 0xc3, 0x5b, 0xb3, 0xa9, 0x6d, 0x0a, 0xe9, 0x10, 0xf6, 0x8d, 0x1e, 0x86, - 0xca, 0x8e, 0x4d, 0x76, 0xac, 0x5d, 0xdc, 0x17, 0xcf, 0xbe, 0x9a, 0x59, 0x66, 0x80, 0x97, 0x71, - 0x9f, 0xa0, 0x87, 0xa0, 0xec, 0xf7, 0x3a, 0xe2, 0x82, 0x95, 0x96, 0xb4, 0x95, 0x9a, 0x59, 0xf2, - 0x7b, 0x1d, 0x7e, 0xbd, 0xfe, 0x9a, 0x83, 0x99, 0xbb, 0x3d, 0x16, 0xcf, 0xf0, 0x1c, 0x78, 0xcf, - 0xa3, 0x0f, 0xa6, 0x8c, 0x17, 0x21, 0x2f, 0xde, 0x0c, 0x8c, 0xa2, 0xa1, 0x64, 0x7c, 0x7d, 0x8d, - 0x98, 0x0c, 0x89, 0xd7, 0x99, 0x7a, 0xad, 0x96, 0x7c, 0x64, 0xe5, 0x39, 0xb3, 0x15, 0x06, 0xe1, - 0x1a, 0xc7, 0xb6, 0x82, 0xc3, 0x30, 0x7e, 0x82, 0xf1, 0xad, 0xe0, 0x30, 0x14, 0x83, 0x06, 0xe8, - 0x76, 0x6b, 0xd7, 0x0f, 0xf6, 0x3c, 0xec, 0xb4, 0xb1, 0xc3, 0x8f, 0xbd, 0x6c, 0xa6, 0x60, 0x42, - 0x31, 0xd8, 0xc1, 0x5b, 0x2d, 0x9f, 0xf2, 0x40, 0x22, 0xcf, 0x14, 0x83, 0x41, 0x6e, 0xfa, 0x94, - 0x0d, 0x3b, 0xd8, 0xc3, 0x14, 0xf3, 0xe1, 0x92, 0x18, 0x16, 0x10, 0x39, 0xdc, 0xeb, 0xc6, 0xd4, - 0x65, 0x31, 0x2c, 0x20, 0x6c, 0xf8, 0x0c, 0x54, 0x06, 0x49, 0xee, 0xca, 0x20, 0x1b, 0x28, 0x52, - 0xdb, 0xf7, 0xa0, 0xbe, 0xe1, 0xd9, 0x2d, 0xbc, 0x13, 0x78, 0x0e, 0x0e, 0xb9, 0xf7, 0x43, 0x75, - 0xc8, 0x53, 0xbb, 0x2d, 0xdd, 0x2b, 0xfb, 0x44, 0xcf, 0xca, 0x18, 0x47, 0x5c, 0xdc, 0x2f, 0x28, - 0xfd, 0x50, 0x62, 0x9a, 0x44, 0xea, 0x70, 0x11, 0x8a, 0xbc, 0x34, 0x23, 0x1c, 0xaf, 0x6e, 0xca, - 0x3f, 0xe3, 0xed, 0xd4, 0xba, 0xb7, 0xc2, 0xa0, 0xd7, 0x45, 0xeb, 0xa0, 0x77, 0x07, 0x30, 0x76, - 0x9a, 0xd9, 0x5e, 0x6f, 0x98, 0x69, 0x33, 0x45, 0x6a, 0x7c, 0x9a, 0x87, 0xda, 0x26, 0xb6, 0xc3, - 0xd6, 0xce, 0x49, 0x48, 0x36, 0x30, 0x89, 0x3b, 0xc4, 0x93, 0x26, 0x81, 0x7d, 0xa2, 0x4b, 0x70, - 0x2a, 0xb1, 0x21, 0xab, 0xcd, 0x04, 0xc4, 0x35, 0x43, 0x37, 0xeb, 0xdd, 0x61, 0xc1, 0x3d, 0x03, - 0x65, 0x87, 0x78, 0x16, 0x3f, 0xa2, 0x12, 0x3f, 0x22, 0xf5, 0xfe, 0xd6, 0x88, 0xc7, 0x8f, 0xa6, - 0xe4, 0x88, 0x0f, 0xf4, 0x28, 0xd4, 0x82, 0x1e, 0xed, 0xf6, 0xa8, 0x25, 0x6e, 0x66, 0xa3, 0xcc, - 0xd9, 0xd3, 0x05, 0x90, 0x5f, 0x5c, 0x82, 0x5e, 0x82, 0x1a, 0xe1, 0xa2, 0x8c, 0xde, 0xa6, 0x95, - 0x71, 0x9f, 0x50, 0xba, 0xa0, 0x13, 0x8f, 0x53, 0x74, 0x01, 0xea, 0x34, 0xb4, 0xef, 0x61, 0x2f, - 0x51, 0x74, 0x01, 0xae, 0x8f, 0xb3, 0x02, 0x3e, 0x28, 0xd2, 0x5c, 0x81, 0xb9, 0x76, 0xcf, 0x0e, - 0x6d, 0x9f, 0x62, 0x9c, 0xc0, 0xae, 0x72, 0x6c, 0x14, 0x0f, 0x0d, 0x2a, 0x34, 0x2f, 0xc3, 0xf4, - 0x6d, 0x97, 0x72, 0x41, 0xb2, 0x5b, 0xad, 0xf1, 0x48, 0x80, 0xdf, 0xdd, 0x87, 0xa0, 0x1c, 0x06, - 0x7b, 0xc2, 0x4a, 0xe5, 0xb8, 0x0a, 0x96, 0xc2, 0x60, 0x8f, 0x9b, 0x20, 0x5e, 0x56, 0x0e, 0x42, - 0xa9, 0x9b, 0x39, 0x53, 0xfe, 0x19, 0xdf, 0xd5, 0x06, 0xca, 0xc3, 0x0c, 0x0c, 0x79, 0x30, 0x0b, - 0xf3, 0x02, 0x94, 0x42, 0x41, 0x3f, 0xb2, 0xc8, 0x96, 0x5c, 0x89, 0x5b, 0xc9, 0x88, 0xca, 0xf8, - 0x8e, 0x06, 0xfa, 0x4b, 0x5e, 0x8f, 0x7c, 0x1e, 0x3a, 0xac, 0x4a, 0xab, 0xe7, 0xd5, 0x29, 0xfd, - 0x1f, 0xe6, 0xa0, 0x26, 0xd9, 0x98, 0xc4, 0xfb, 0x67, 0xb2, 0xb2, 0x09, 0x55, 0xb6, 0xa4, 0x45, - 0x70, 0x3b, 0xca, 0x49, 0x54, 0x57, 0x57, 0x95, 0xb7, 0x3e, 0xc5, 0x06, 0x2f, 0x4f, 0x6e, 0x72, - 0xa2, 0xaf, 0xfb, 0x34, 0xec, 0x9b, 0xd0, 0x8a, 0x01, 0xcd, 0xb7, 0x61, 0x76, 0x68, 0x98, 0xe9, - 0xc6, 0x2e, 0xee, 0x47, 0x66, 0x6d, 0x17, 0xf7, 0xd1, 0x53, 0xc9, 0x22, 0x72, 0x96, 0xfb, 0xba, - 0x13, 0xf8, 0xed, 0xeb, 0x61, 0x68, 0xf7, 0x65, 0x91, 0xf9, 0xb9, 0xdc, 0xb3, 0x9a, 0xf1, 0xe7, - 0x1c, 0xe8, 0xaf, 0xf6, 0x70, 0xd8, 0x3f, 0x4a, 0xf3, 0x82, 0x60, 0x1a, 0xdf, 0xef, 0x86, 0xd2, - 0x41, 0xf3, 0xef, 0xfd, 0x37, 0xba, 0xa0, 0xb8, 0xd1, 0x0a, 0xbb, 0x54, 0x54, 0xda, 0x25, 0xd5, - 0x95, 0x2d, 0x1d, 0xea, 0xca, 0x96, 0x33, 0xaf, 0x2c, 0xd3, 0x6e, 0x29, 0xc2, 0x89, 0x2e, 0x59, - 0xea, 0x1d, 0x92, 0x3b, 0xec, 0x3b, 0xc4, 0xf8, 0x58, 0x83, 0xca, 0x1b, 0xb8, 0x45, 0x83, 0x90, - 0x59, 0x0b, 0x85, 0xec, 0xb5, 0x31, 0x9e, 0x7a, 0xb9, 0xe1, 0xa7, 0xde, 0x35, 0x28, 0xbb, 0x8e, - 0x65, 0x33, 0xb5, 0xe1, 0x87, 0x37, 0xea, 0x89, 0x51, 0x72, 0x1d, 0xae, 0x5f, 0xe3, 0xe7, 0xa6, - 0x7f, 0xaa, 0x81, 0x2e, 0x78, 0x26, 0x82, 0xf2, 0xf9, 0xc4, 0x72, 0x9a, 0x4a, 0x97, 0xe5, 0x4f, - 0xbc, 0xd1, 0xdb, 0x53, 0x83, 0x65, 0xaf, 0x03, 0x30, 0xd9, 0x49, 0x72, 0x71, 0x15, 0x96, 0x94, - 0xdc, 0x0a, 0x72, 0x2e, 0xc7, 0xdb, 0x53, 0x66, 0x85, 0x51, 0xf1, 0x29, 0x6e, 0x94, 0xa0, 0xc0, - 0xa9, 0x8d, 0xff, 0x68, 0x30, 0x77, 0xd3, 0xf6, 0x5a, 0x6b, 0x2e, 0xa1, 0xb6, 0xdf, 0x9a, 0x20, - 0xf6, 0x79, 0x0e, 0x4a, 0x41, 0xd7, 0xf2, 0xf0, 0x36, 0x95, 0x2c, 0x2d, 0x8f, 0xd8, 0x91, 0x10, - 0x83, 0x59, 0x0c, 0xba, 0x77, 0xf0, 0x36, 0x45, 0x5f, 0x81, 0x72, 0xd0, 0xb5, 0x42, 0xb7, 0xbd, - 0x43, 0xa5, 0xf4, 0xc7, 0x20, 0x2e, 0x05, 0x5d, 0x93, 0x51, 0x24, 0x72, 0x05, 0xd3, 0x87, 0xcc, - 0x15, 0x18, 0xff, 0xd8, 0xb7, 0xfd, 0x09, 0x54, 0xfb, 0x39, 0x28, 0xbb, 0x3e, 0xb5, 0x1c, 0x97, - 0x44, 0x22, 0x38, 0xab, 0xd6, 0x21, 0x9f, 0xf2, 0x1d, 0xf0, 0x33, 0xf5, 0x29, 0x5b, 0x1b, 0xbd, - 0x08, 0xb0, 0xed, 0x05, 0xb6, 0xa4, 0x16, 0x32, 0x38, 0xa7, 0xbe, 0x15, 0x0c, 0x2d, 0xa2, 0xaf, - 0x70, 0x22, 0x36, 0xc3, 0xe0, 0x48, 0xff, 0xa6, 0xc1, 0xc2, 0x06, 0x0e, 0x89, 0x4b, 0x28, 0xf6, - 0xa9, 0xcc, 0xdb, 0xad, 0xfb, 0xdb, 0x41, 0x3a, 0x41, 0xaa, 0x0d, 0x25, 0x48, 0x3f, 0x9b, 0x74, - 0x61, 0x2a, 0x12, 0x10, 0x69, 0xfa, 0x28, 0x12, 0x88, 0x8a, 0x11, 0x22, 0x92, 0x9a, 0xc9, 0x38, - 0x26, 0xc9, 0x6f, 0x32, 0xa0, 0x34, 0x7e, 0x24, 0x1a, 0x03, 0x94, 0x9b, 0x7a, 0x70, 0x85, 0x5d, - 0x04, 0x69, 0xc0, 0x87, 0xcc, 0xf9, 0x17, 0x61, 0xc8, 0x76, 0x64, 0xb4, 0x2b, 0xfc, 0x4c, 0x83, - 0xa5, 0x6c, 0xae, 0x26, 0xf1, 0xbc, 0x2f, 0x42, 0xc1, 0xf5, 0xb7, 0x83, 0x28, 0x8d, 0x74, 0x51, - 0xfd, 0xa0, 0x56, 0xae, 0x2b, 0x08, 0x8d, 0x7f, 0x6b, 0x50, 0xe7, 0xb6, 0xfa, 0x08, 0x8e, 0xbf, - 0x83, 0x3b, 0x16, 0x71, 0xdf, 0xc1, 0xd1, 0xf1, 0x77, 0x70, 0x67, 0xd3, 0x7d, 0x07, 0xa7, 0x34, - 0xa3, 0x90, 0xd6, 0x8c, 0x74, 0xa0, 0x5d, 0x1c, 0x91, 0x26, 0x2c, 0xa5, 0xd2, 0x84, 0xc6, 0x07, - 0x1a, 0x34, 0x6f, 0x61, 0x3a, 0xbc, 0xd5, 0xa3, 0x53, 0x8a, 0x0f, 0x35, 0x78, 0x58, 0xc9, 0xd0, - 0x24, 0xfa, 0xf0, 0x7c, 0x5a, 0x1f, 0xd4, 0x01, 0xd6, 0xbe, 0x25, 0xa5, 0x2a, 0x5c, 0x05, 0x7d, - 0xad, 0xd7, 0xe9, 0xc4, 0x0f, 0x9f, 0x65, 0xd0, 0x43, 0xf1, 0x29, 0xe2, 0x0f, 0xe1, 0x2e, 0xab, - 0x12, 0xc6, 0xa2, 0x0c, 0xe3, 0x12, 0xd4, 0x24, 0x89, 0xe4, 0xba, 0x09, 0xe5, 0x50, 0x7e, 0x4b, - 0xfc, 0xf8, 0xdf, 0x58, 0x80, 0x39, 0x13, 0xb7, 0x99, 0x26, 0x86, 0x77, 0x5c, 0x7f, 0x57, 0x2e, - 0x63, 0xbc, 0xab, 0xc1, 0x7c, 0x1a, 0x2e, 0xe7, 0xfa, 0x12, 0x94, 0x6c, 0xc7, 0x09, 0x31, 0x21, - 0x23, 0x8f, 0xe5, 0xba, 0xc0, 0x31, 0x23, 0xe4, 0x84, 0xe4, 0x72, 0x63, 0x4b, 0xce, 0xb0, 0xe0, - 0xd4, 0x2d, 0x4c, 0xef, 0x62, 0x1a, 0x4e, 0x54, 0x07, 0x6e, 0xb0, 0xc8, 0x80, 0x13, 0x4b, 0xb5, - 0x88, 0x7e, 0x8d, 0xf7, 0x35, 0x40, 0xc9, 0x15, 0x26, 0x39, 0xe6, 0xa4, 0x94, 0x73, 0x69, 0x29, - 0x8b, 0x56, 0x99, 0x4e, 0x37, 0xf0, 0xb1, 0x4f, 0x93, 0x4f, 0xcc, 0x5a, 0x0c, 0x65, 0xea, 0x77, - 0x71, 0x19, 0xca, 0x51, 0xe9, 0x12, 0x95, 0x20, 0x7f, 0xdd, 0xf3, 0xea, 0x53, 0x48, 0x87, 0xf2, - 0xba, 0xac, 0xcf, 0xd5, 0xb5, 0x8b, 0x5f, 0x83, 0xd9, 0xa1, 0xc8, 0x1f, 0x95, 0x61, 0xfa, 0x95, - 0xc0, 0xc7, 0xf5, 0x29, 0x54, 0x07, 0xfd, 0x86, 0xeb, 0xdb, 0x61, 0x5f, 0x78, 0xda, 0xba, 0x83, - 0x66, 0xa1, 0xca, 0x3d, 0x8e, 0x04, 0xe0, 0xd5, 0x9f, 0x9f, 0x86, 0xda, 0x5d, 0xbe, 0x99, 0x4d, - 0x1c, 0xde, 0x73, 0x5b, 0x18, 0x59, 0x50, 0x1f, 0x6e, 0x3e, 0x46, 0x8f, 0x2b, 0x75, 0x34, 0xa3, - 0x47, 0xb9, 0x39, 0x4a, 0x3c, 0xc6, 0x14, 0x7a, 0x0b, 0x66, 0xd2, 0x6d, 0xc1, 0x48, 0x6d, 0x12, - 0x95, 0xbd, 0xc3, 0x07, 0x4d, 0x6e, 0x41, 0x2d, 0xd5, 0xe5, 0x8b, 0x2e, 0x28, 0xe7, 0x56, 0x75, - 0x02, 0x37, 0xd5, 0xaf, 0x94, 0x64, 0x27, 0xae, 0xe0, 0x3e, 0xdd, 0x8b, 0x98, 0xc1, 0xbd, 0xb2, - 0x61, 0xf1, 0x20, 0xee, 0x6d, 0x38, 0xb5, 0xaf, 0xb5, 0x10, 0x3d, 0xa1, 0x9c, 0x3f, 0xab, 0x05, - 0xf1, 0xa0, 0x25, 0xf6, 0x00, 0xed, 0xef, 0x66, 0x45, 0x97, 0xd5, 0x27, 0x90, 0xd5, 0xcb, 0xdb, - 0xbc, 0x32, 0x36, 0x7e, 0x2c, 0xb8, 0xef, 0x69, 0x70, 0x3a, 0xa3, 0x1f, 0x10, 0x5d, 0x53, 0x4e, - 0x37, 0xba, 0xa9, 0xb1, 0xf9, 0xd4, 0xe1, 0x88, 0x62, 0x46, 0x7c, 0x98, 0x1d, 0x6a, 0x91, 0x43, - 0x97, 0x32, 0xdb, 0x06, 0xf6, 0xf7, 0x0a, 0x36, 0x1f, 0x1f, 0x0f, 0x39, 0x5e, 0x8f, 0xc5, 0xc2, - 0xe9, 0xbe, 0xb2, 0x8c, 0xf5, 0xd4, 0xdd, 0x67, 0x07, 0x1d, 0xe8, 0x9b, 0x50, 0x4b, 0x35, 0x80, - 0x65, 0x68, 0xbc, 0xaa, 0x49, 0xec, 0xa0, 0xa9, 0xdf, 0x06, 0x3d, 0xd9, 0xa7, 0x85, 0x56, 0xb2, - 0xee, 0xd2, 0xbe, 0x89, 0x0f, 0x73, 0x95, 0x06, 0x6d, 0x18, 0x23, 0xae, 0xd2, 0xbe, 0xce, 0x95, - 0xf1, 0xaf, 0x52, 0x62, 0xfe, 0x91, 0x57, 0xe9, 0xd0, 0x4b, 0xbc, 0xab, 0xc1, 0xa2, 0xba, 0xcd, - 0x07, 0xad, 0x66, 0xe9, 0x66, 0x76, 0x43, 0x53, 0xf3, 0xda, 0xa1, 0x68, 0x62, 0x29, 0xee, 0xc2, - 0x4c, 0xba, 0x99, 0x25, 0x43, 0x8a, 0xca, 0xfe, 0x9f, 0xe6, 0xa5, 0xb1, 0x70, 0xe3, 0xc5, 0x5e, - 0x87, 0x6a, 0xa2, 0xa0, 0x8f, 0x1e, 0x1b, 0xa1, 0xc7, 0xc9, 0x72, 0xd0, 0x41, 0x92, 0xdc, 0x81, - 0x5a, 0xaa, 0x88, 0x9b, 0xa5, 0xc3, 0x8a, 0xda, 0x7a, 0xf3, 0xe2, 0x38, 0xa8, 0xf1, 0x06, 0x76, - 0xa0, 0x96, 0x2a, 0xa9, 0x65, 0xac, 0xa4, 0xaa, 0x20, 0x66, 0xac, 0xa4, 0xac, 0xd0, 0x19, 0x53, - 0xe8, 0xdb, 0x89, 0xea, 0x5d, 0xaa, 0x42, 0x8a, 0xae, 0x8e, 0x9c, 0x47, 0x55, 0x20, 0x6e, 0xae, - 0x1e, 0x86, 0x24, 0x66, 0xe1, 0x55, 0xa8, 0xc4, 0x85, 0x39, 0x74, 0x3e, 0xd3, 0x2c, 0x1c, 0xe6, - 0xa4, 0x36, 0xa1, 0x28, 0x8a, 0x64, 0xc8, 0xc8, 0x28, 0x87, 0x27, 0x2a, 0x68, 0xcd, 0x47, 0x95, - 0x38, 0xe9, 0xfa, 0x91, 0x31, 0x85, 0x4c, 0x28, 0x8a, 0x34, 0x6c, 0xc6, 0xa4, 0xa9, 0x52, 0x42, - 0x73, 0x34, 0x8e, 0xc8, 0xdd, 0x4e, 0xa1, 0x0d, 0x28, 0xf0, 0x74, 0x25, 0x5a, 0x1e, 0x95, 0xca, - 0x1c, 0x35, 0x63, 0x2a, 0xdb, 0x69, 0x4c, 0xa1, 0x6f, 0x40, 0x81, 0xbf, 0xca, 0x33, 0x66, 0x4c, - 0xe6, 0x23, 0x9b, 0x23, 0x51, 0x22, 0x16, 0x1d, 0xd0, 0x93, 0xd9, 0x8a, 0x0c, 0xf3, 0xaa, 0xc8, - 0xe7, 0x34, 0xc7, 0xc1, 0x8c, 0x56, 0xf9, 0xbe, 0x06, 0x8d, 0xac, 0xc0, 0x16, 0x65, 0xfa, 0xd0, - 0x51, 0xd1, 0x79, 0xf3, 0xe9, 0x43, 0x52, 0xc5, 0x22, 0x7c, 0x07, 0xe6, 0x14, 0xe1, 0x14, 0xba, - 0x92, 0x35, 0x5f, 0x46, 0x24, 0xd8, 0x7c, 0x72, 0x7c, 0x82, 0x78, 0xed, 0x0d, 0x28, 0xf0, 0x30, - 0x28, 0xe3, 0xf8, 0x92, 0x51, 0x55, 0x86, 0x42, 0xa4, 0xa2, 0x28, 0x63, 0x0a, 0x61, 0xd0, 0x93, - 0x31, 0x51, 0xc6, 0xf9, 0x29, 0xc2, 0xa9, 0xe6, 0x85, 0x31, 0x30, 0xa3, 0x65, 0x56, 0x7b, 0xa0, - 0x6f, 0x84, 0xc1, 0xfd, 0x7e, 0xf4, 0x40, 0xff, 0xff, 0x2c, 0x7b, 0xe3, 0xe9, 0x6f, 0x5e, 0x6b, - 0xbb, 0x74, 0xa7, 0xb7, 0xc5, 0x6c, 0xc0, 0x15, 0x81, 0xfb, 0x84, 0x1b, 0xc8, 0xaf, 0x2b, 0xae, - 0x4f, 0x71, 0xe8, 0xdb, 0xde, 0x15, 0x3e, 0x97, 0x84, 0x76, 0xb7, 0xb6, 0x8a, 0xfc, 0xff, 0xda, - 0xff, 0x02, 0x00, 0x00, 0xff, 0xff, 0x6d, 0x26, 0xc3, 0xaf, 0x04, 0x39, 0x00, 0x00, + // 2993 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xd4, 0x3a, 0xcd, 0x73, 0x1c, 0x47, + 0xf5, 0x9a, 0x5d, 0xed, 0xd7, 0xdb, 0x59, 0x69, 0xdd, 0xfa, 0xf0, 0x66, 0x63, 0xc7, 0xd2, 0xe4, + 0xe7, 0x44, 0xb6, 0x13, 0x3b, 0x96, 0x93, 0x5f, 0x42, 0x02, 0x24, 0xb6, 0x45, 0x6c, 0x55, 0xec, + 0xa0, 0x8c, 0x92, 0x54, 0x85, 0x54, 0x6a, 0x6a, 0xb4, 0xd3, 0x5a, 0x4d, 0x69, 0x76, 0x66, 0x99, + 0xee, 0xb5, 0xbc, 0x39, 0x41, 0x05, 0xa8, 0xa2, 0x12, 0x92, 0xa2, 0xa0, 0xa0, 0xb8, 0x02, 0x39, + 0x70, 0x23, 0x40, 0x15, 0x14, 0x27, 0x0e, 0x39, 0x70, 0xa0, 0x8a, 0x8f, 0xbf, 0x80, 0x0b, 0xc7, + 0xfc, 0x07, 0x54, 0x41, 0xf5, 0xc7, 0xcc, 0xce, 0xac, 0x7a, 0x56, 0x2b, 0x6f, 0x82, 0xa4, 0xdb, + 0xcc, 0xeb, 0xf7, 0xba, 0x5f, 0xbf, 0x7e, 0xfd, 0x5e, 0xbf, 0x0f, 0xd0, 0x3b, 0xae, 0x77, 0xaf, + 0x47, 0x2e, 0x77, 0xc3, 0x80, 0x06, 0x68, 0x2e, 0xf9, 0x77, 0x59, 0xfc, 0x34, 0xf5, 0x56, 0xd0, + 0xe9, 0x04, 0xbe, 0x00, 0x36, 0x75, 0xd2, 0xda, 0xc1, 0x1d, 0x5b, 0xfc, 0x19, 0x9f, 0x6a, 0x70, + 0xfa, 0x66, 0x88, 0x6d, 0x8a, 0x6f, 0x06, 0x9e, 0x87, 0x5b, 0xd4, 0x0d, 0x7c, 0x13, 0x7f, 0xb3, + 0x87, 0x09, 0x45, 0x4f, 0xc1, 0xf4, 0x96, 0x4d, 0x70, 0x43, 0x5b, 0xd2, 0x56, 0xaa, 0xab, 0x67, + 0x2e, 0xa7, 0xe6, 0x96, 0x73, 0xde, 0x25, 0xed, 0x1b, 0x36, 0xc1, 0x26, 0xc7, 0x44, 0xa7, 0xa1, + 0xe4, 0x6c, 0x59, 0xbe, 0xdd, 0xc1, 0x8d, 0xdc, 0x92, 0xb6, 0x52, 0x31, 0x8b, 0xce, 0xd6, 0xab, + 0x76, 0x07, 0xa3, 0xc7, 0x61, 0xb6, 0x15, 0xcf, 0x2f, 0x10, 0xf2, 0x1c, 0x61, 0x66, 0x00, 0xe6, + 0x88, 0x8b, 0x50, 0x14, 0xfc, 0x35, 0xa6, 0x97, 0xb4, 0x15, 0xdd, 0x94, 0x7f, 0xe8, 0x2c, 0x00, + 0xd9, 0xb1, 0x43, 0x87, 0x58, 0x7e, 0xaf, 0xd3, 0x28, 0x2c, 0x69, 0x2b, 0x05, 0xb3, 0x22, 0x20, + 0xaf, 0xf6, 0x3a, 0xc6, 0xfb, 0x1a, 0x2c, 0xac, 0x85, 0x41, 0xf7, 0x58, 0x6c, 0xc2, 0xf8, 0x95, + 0x06, 0xf3, 0xb7, 0x6d, 0x72, 0x3c, 0x24, 0x7a, 0x16, 0x80, 0xba, 0x1d, 0x6c, 0x11, 0x6a, 0x77, + 0xba, 0x5c, 0xaa, 0xd3, 0x66, 0x85, 0x41, 0x36, 0x19, 0xc0, 0x78, 0x0b, 0xf4, 0x1b, 0x41, 0xe0, + 0x99, 0x98, 0x74, 0x03, 0x9f, 0x60, 0x74, 0x0d, 0x8a, 0x84, 0xda, 0xb4, 0x47, 0x24, 0x93, 0x0f, + 0x2b, 0x99, 0xdc, 0xe4, 0x28, 0xa6, 0x44, 0x45, 0xf3, 0x50, 0xb8, 0x67, 0x7b, 0x3d, 0xc1, 0x63, + 0xd9, 0x14, 0x3f, 0xc6, 0xdb, 0x30, 0xb3, 0x49, 0x43, 0xd7, 0x6f, 0x7f, 0x8e, 0x93, 0x57, 0xa2, + 0xc9, 0xff, 0xa1, 0xc1, 0x43, 0x6b, 0x98, 0xb4, 0x42, 0x77, 0xeb, 0x98, 0xa8, 0xae, 0x01, 0xfa, + 0x00, 0xb2, 0xbe, 0xc6, 0x45, 0x9d, 0x37, 0x53, 0xb0, 0xa1, 0xc3, 0x28, 0x0c, 0x1f, 0xc6, 0x7f, + 0x72, 0xd0, 0x54, 0x6d, 0x6a, 0x12, 0xf1, 0x7d, 0x25, 0xbe, 0x51, 0x39, 0x4e, 0x74, 0x3e, 0x4d, + 0x24, 0xad, 0xc1, 0x60, 0xb5, 0x4d, 0x0e, 0x88, 0x2f, 0xde, 0xf0, 0xae, 0xf2, 0x8a, 0x5d, 0xad, + 0xc2, 0xc2, 0x3d, 0x37, 0xa4, 0x3d, 0xdb, 0xb3, 0x5a, 0x3b, 0xb6, 0xef, 0x63, 0x8f, 0xcb, 0x89, + 0x34, 0xa6, 0x97, 0xf2, 0x2b, 0x15, 0x73, 0x4e, 0x0e, 0xde, 0x14, 0x63, 0x4c, 0x58, 0x04, 0x3d, + 0x0d, 0x8b, 0xdd, 0x9d, 0x3e, 0x71, 0x5b, 0xfb, 0x88, 0x0a, 0x9c, 0x68, 0x3e, 0x1a, 0x4d, 0x51, + 0x5d, 0x82, 0x53, 0x2d, 0x6e, 0xad, 0x1c, 0x8b, 0x49, 0x4d, 0x88, 0xb1, 0xc8, 0xc5, 0x58, 0x97, + 0x03, 0xaf, 0x47, 0x70, 0xc6, 0x56, 0x84, 0xdc, 0xa3, 0xad, 0x04, 0x41, 0x89, 0x13, 0xcc, 0xc9, + 0xc1, 0x37, 0x68, 0x2b, 0xa6, 0xe1, 0x86, 0xe4, 0x4e, 0x60, 0x3b, 0xc7, 0xc3, 0x90, 0x7c, 0xa8, + 0x41, 0xc3, 0xc4, 0x1e, 0xb6, 0xc9, 0xf1, 0xd0, 0x71, 0xe3, 0xc7, 0x1a, 0x3c, 0x72, 0x0b, 0xd3, + 0x84, 0xb6, 0x50, 0x9b, 0xba, 0x84, 0xba, 0x2d, 0x72, 0x94, 0x6c, 0x7d, 0xa4, 0xc1, 0xb9, 0x4c, + 0xb6, 0x26, 0xb9, 0x3c, 0xcf, 0x42, 0x81, 0x7d, 0x91, 0x46, 0x6e, 0x29, 0xbf, 0x52, 0x5d, 0x5d, + 0x56, 0xd2, 0xbc, 0x82, 0xfb, 0x6f, 0x32, 0x9b, 0xb4, 0x61, 0xbb, 0xa1, 0x29, 0xf0, 0x8d, 0x7f, + 0x6a, 0xb0, 0xb8, 0xb9, 0x13, 0xec, 0x0d, 0x58, 0xfa, 0x22, 0x04, 0x94, 0x36, 0x27, 0xf9, 0x21, + 0x73, 0x82, 0xae, 0xc2, 0x34, 0xed, 0x77, 0x31, 0xb7, 0x44, 0x33, 0xab, 0x67, 0x2f, 0x2b, 0x1e, + 0x07, 0x97, 0x19, 0x93, 0xaf, 0xf7, 0xbb, 0xd8, 0xe4, 0xa8, 0xe8, 0x02, 0xd4, 0x87, 0x44, 0x1e, + 0x5d, 0xc8, 0xd9, 0xb4, 0xcc, 0x89, 0xf1, 0x87, 0x1c, 0x9c, 0xde, 0xb7, 0xc5, 0x49, 0x84, 0xad, + 0x5a, 0x3b, 0xa7, 0x5c, 0x1b, 0x9d, 0x87, 0x84, 0x0a, 0x58, 0xae, 0x43, 0x1a, 0xf9, 0xa5, 0xfc, + 0x4a, 0xde, 0xac, 0x25, 0xec, 0x92, 0x43, 0xd0, 0x93, 0x80, 0xf6, 0x99, 0x0b, 0x61, 0x95, 0xa6, + 0xcd, 0x53, 0xc3, 0xf6, 0x82, 0xdb, 0x24, 0xa5, 0xc1, 0x10, 0x22, 0x98, 0x36, 0xe7, 0x15, 0x16, + 0x83, 0xa0, 0xab, 0x30, 0xef, 0xfa, 0x77, 0x71, 0x27, 0x08, 0xfb, 0x56, 0x17, 0x87, 0x2d, 0xec, + 0x53, 0xbb, 0x8d, 0x49, 0xa3, 0xc8, 0x39, 0x9a, 0x8b, 0xc6, 0x36, 0x06, 0x43, 0xc6, 0x6f, 0x35, + 0x58, 0x14, 0xaf, 0xae, 0x0d, 0x3b, 0xa4, 0xee, 0x51, 0x7b, 0xae, 0xf3, 0x30, 0xd3, 0x8d, 0xf8, + 0x10, 0x78, 0xd3, 0x1c, 0xaf, 0x16, 0x43, 0xf9, 0x2d, 0xfb, 0x44, 0x83, 0x79, 0xf6, 0xc8, 0x3a, + 0x49, 0x3c, 0xff, 0x5a, 0x83, 0xb9, 0xdb, 0x36, 0x39, 0x49, 0x2c, 0xff, 0x4e, 0xba, 0xa0, 0x98, + 0xe7, 0xa3, 0x34, 0xad, 0x0c, 0x31, 0xcd, 0x74, 0xe4, 0xd5, 0x67, 0x52, 0x5c, 0x13, 0xe3, 0xf7, + 0x03, 0x5f, 0x75, 0xc2, 0x38, 0xff, 0xa3, 0x06, 0x67, 0x6f, 0x61, 0x1a, 0x73, 0x7d, 0x2c, 0x7c, + 0xda, 0xb8, 0xda, 0xf2, 0xa1, 0xf0, 0xc8, 0x4a, 0xe6, 0x8f, 0xc4, 0xf3, 0xbd, 0x9f, 0x83, 0x05, + 0xe6, 0x16, 0x8e, 0x87, 0x12, 0x8c, 0xf3, 0x28, 0x57, 0x28, 0x4a, 0x41, 0xa5, 0x28, 0xb1, 0x3f, + 0x2d, 0x8e, 0xed, 0x4f, 0x8d, 0xdf, 0xe4, 0xc4, 0x3b, 0x20, 0x29, 0x8d, 0x49, 0x8e, 0x45, 0xc1, + 0x6b, 0x4e, 0xc9, 0xab, 0x01, 0x7a, 0x0c, 0x59, 0x5f, 0x8b, 0xfc, 0x63, 0x0a, 0x76, 0x6c, 0xdd, + 0xe3, 0x07, 0x1a, 0x2c, 0x46, 0x61, 0xd0, 0x26, 0x6e, 0x77, 0xb0, 0x4f, 0x1f, 0x5c, 0x87, 0x86, + 0x35, 0x20, 0xa7, 0xd0, 0x80, 0x33, 0x50, 0x21, 0x62, 0x9d, 0x38, 0xc2, 0x19, 0x00, 0x8c, 0x8f, + 0x35, 0x38, 0xbd, 0x8f, 0x9d, 0x49, 0x0e, 0xb1, 0x01, 0x25, 0xd7, 0x77, 0xf0, 0xfd, 0x98, 0x9b, + 0xe8, 0x97, 0x8d, 0x6c, 0xf5, 0x5c, 0xcf, 0x89, 0xd9, 0x88, 0x7e, 0xd1, 0x32, 0xe8, 0xd8, 0xb7, + 0xb7, 0x3c, 0x6c, 0x71, 0x5c, 0xae, 0xc8, 0x65, 0xb3, 0x2a, 0x60, 0xeb, 0x0c, 0x64, 0xfc, 0x40, + 0x83, 0x39, 0xa6, 0x6b, 0x92, 0x47, 0xf2, 0xc5, 0xca, 0x6c, 0x09, 0xaa, 0x09, 0x65, 0x92, 0xec, + 0x26, 0x41, 0xc6, 0x2e, 0xcc, 0xa7, 0xd9, 0x99, 0x44, 0x66, 0x8f, 0x00, 0xc4, 0x27, 0x22, 0x74, + 0x3e, 0x6f, 0x26, 0x20, 0xc6, 0x67, 0x1a, 0x20, 0xf1, 0xa4, 0xe2, 0xc2, 0x38, 0xe2, 0x8c, 0xcb, + 0xb6, 0x8b, 0x3d, 0x27, 0x69, 0xb5, 0x2b, 0x1c, 0xc2, 0x87, 0xd7, 0x40, 0xc7, 0xf7, 0x69, 0x68, + 0x5b, 0x5d, 0x3b, 0xb4, 0x3b, 0xe2, 0xf2, 0x8c, 0x65, 0x60, 0xab, 0x9c, 0x6c, 0x83, 0x53, 0x19, + 0x7f, 0x66, 0x8f, 0x31, 0xa9, 0x94, 0xc7, 0x7d, 0xc7, 0x67, 0x01, 0xb8, 0xd2, 0x8a, 0xe1, 0x82, + 0x18, 0xe6, 0x10, 0xee, 0xc2, 0x3e, 0xd6, 0xa0, 0xce, 0xb7, 0x20, 0xf6, 0xd3, 0x65, 0xd3, 0x0e, + 0xd1, 0x68, 0x43, 0x34, 0x23, 0xae, 0xd0, 0x97, 0xa0, 0x28, 0x05, 0x9b, 0x1f, 0x57, 0xb0, 0x92, + 0xe0, 0x80, 0x6d, 0x18, 0x3f, 0xd7, 0x60, 0x61, 0x48, 0xe4, 0x93, 0x68, 0xf4, 0xeb, 0x80, 0xc4, + 0x0e, 0x9d, 0xc1, 0xb6, 0x23, 0x77, 0x7b, 0x5e, 0xe9, 0x5b, 0x86, 0x85, 0x64, 0x9e, 0x72, 0x87, + 0x20, 0xc4, 0xf8, 0x9b, 0x06, 0x67, 0x6e, 0x61, 0xca, 0x51, 0x6f, 0x30, 0xdb, 0xb1, 0x11, 0x06, + 0xed, 0x10, 0x13, 0x72, 0x72, 0xf5, 0xe3, 0x27, 0xe2, 0x7d, 0xa6, 0xda, 0xd2, 0x24, 0xf2, 0x5f, + 0x06, 0x9d, 0xaf, 0x81, 0x1d, 0x2b, 0x0c, 0xf6, 0x88, 0xd4, 0xa3, 0xaa, 0x84, 0x99, 0xc1, 0x1e, + 0x57, 0x08, 0x1a, 0x50, 0xdb, 0x13, 0x08, 0xd2, 0x31, 0x70, 0x08, 0x1b, 0xe6, 0x77, 0x30, 0x62, + 0x8c, 0x4d, 0x8e, 0x4f, 0xae, 0x8c, 0x7f, 0xa9, 0xc1, 0xc2, 0xd0, 0x56, 0x26, 0x91, 0xed, 0x33, + 0xe2, 0xf5, 0x28, 0x36, 0x33, 0xb3, 0x7a, 0x4e, 0x49, 0x93, 0x58, 0x4c, 0x60, 0xa3, 0x73, 0x50, + 0xdd, 0xb6, 0x5d, 0xcf, 0x0a, 0xb1, 0x4d, 0x02, 0x5f, 0x6e, 0x14, 0x18, 0xc8, 0xe4, 0x10, 0xe3, + 0x53, 0x0d, 0xea, 0x2c, 0x04, 0x3d, 0xe1, 0x16, 0xef, 0x17, 0x39, 0xa8, 0xad, 0xfb, 0x04, 0x87, + 0xf4, 0xf8, 0x47, 0x18, 0xe8, 0x45, 0xa8, 0xf2, 0x8d, 0x11, 0xcb, 0xb1, 0xa9, 0x2d, 0xdd, 0xd5, + 0x23, 0xca, 0x2c, 0xf2, 0xcb, 0x0c, 0x6f, 0xcd, 0xa6, 0xb6, 0x29, 0xa4, 0x43, 0xd8, 0x37, 0x7a, + 0x18, 0x2a, 0x3b, 0x36, 0xd9, 0xb1, 0x76, 0x71, 0x5f, 0x3c, 0xfb, 0x6a, 0x66, 0x99, 0x01, 0x5e, + 0xc1, 0x7d, 0x82, 0x1e, 0x82, 0xb2, 0xdf, 0xeb, 0x88, 0x0b, 0x56, 0x5a, 0xd2, 0x56, 0x6a, 0x66, + 0xc9, 0xef, 0x75, 0xf8, 0xf5, 0xfa, 0x4b, 0x0e, 0x66, 0xee, 0xf6, 0x58, 0x3c, 0xc3, 0x73, 0xe0, + 0x3d, 0x8f, 0x3e, 0x98, 0x32, 0x5e, 0x84, 0xbc, 0x78, 0x33, 0x30, 0x8a, 0x86, 0x92, 0xf1, 0xf5, + 0x35, 0x62, 0x32, 0x24, 0x5e, 0x67, 0xea, 0xb5, 0x5a, 0xf2, 0x91, 0x95, 0xe7, 0xcc, 0x56, 0x18, + 0x84, 0x6b, 0x1c, 0xdb, 0x0a, 0x0e, 0xc3, 0xf8, 0x09, 0xc6, 0xb7, 0x82, 0xc3, 0x50, 0x0c, 0x1a, + 0xa0, 0xdb, 0xad, 0x5d, 0x3f, 0xd8, 0xf3, 0xb0, 0xd3, 0xc6, 0x0e, 0x3f, 0xf6, 0xb2, 0x99, 0x82, + 0x09, 0xc5, 0x60, 0x07, 0x6f, 0xb5, 0x7c, 0xca, 0x03, 0x89, 0x3c, 0x53, 0x0c, 0x06, 0xb9, 0xe9, + 0x53, 0x36, 0xec, 0x60, 0x0f, 0x53, 0xcc, 0x87, 0x4b, 0x62, 0x58, 0x40, 0xe4, 0x70, 0xaf, 0x1b, + 0x53, 0x97, 0xc5, 0xb0, 0x80, 0xb0, 0xe1, 0x33, 0x50, 0x19, 0x24, 0xb9, 0x2b, 0x83, 0x6c, 0xa0, + 0x48, 0x6d, 0xdf, 0x83, 0xfa, 0x86, 0x67, 0xb7, 0xf0, 0x4e, 0xe0, 0x39, 0x38, 0xe4, 0xde, 0x0f, + 0xd5, 0x21, 0x4f, 0xed, 0xb6, 0x74, 0xaf, 0xec, 0x13, 0x3d, 0x27, 0x63, 0x1c, 0x71, 0x71, 0xff, + 0x4f, 0xe9, 0x87, 0x12, 0xd3, 0x24, 0x52, 0x87, 0x8b, 0x50, 0xe4, 0xa5, 0x19, 0xe1, 0x78, 0x75, + 0x53, 0xfe, 0x19, 0xef, 0xa4, 0xd6, 0xbd, 0x15, 0x06, 0xbd, 0x2e, 0x5a, 0x07, 0xbd, 0x3b, 0x80, + 0xb1, 0xd3, 0xcc, 0xf6, 0x7a, 0xc3, 0x4c, 0x9b, 0x29, 0x52, 0xe3, 0xb3, 0x3c, 0xd4, 0x36, 0xb1, + 0x1d, 0xb6, 0x76, 0x4e, 0x42, 0xb2, 0x81, 0x49, 0xdc, 0x21, 0x9e, 0x34, 0x09, 0xec, 0x13, 0x5d, + 0x82, 0x53, 0x89, 0x0d, 0x59, 0x6d, 0x26, 0x20, 0xae, 0x19, 0xba, 0x59, 0xef, 0x0e, 0x0b, 0xee, + 0x59, 0x28, 0x3b, 0xc4, 0xb3, 0xf8, 0x11, 0x95, 0xf8, 0x11, 0xa9, 0xf7, 0xb7, 0x46, 0x3c, 0x7e, + 0x34, 0x25, 0x47, 0x7c, 0xa0, 0x47, 0xa1, 0x16, 0xf4, 0x68, 0xb7, 0x47, 0x2d, 0x71, 0x33, 0x1b, + 0x65, 0xce, 0x9e, 0x2e, 0x80, 0xfc, 0xe2, 0x12, 0xf4, 0x32, 0xd4, 0x08, 0x17, 0x65, 0xf4, 0x36, + 0xad, 0x8c, 0xfb, 0x84, 0xd2, 0x05, 0x9d, 0x78, 0x9c, 0xa2, 0x0b, 0x50, 0xa7, 0xa1, 0x7d, 0x0f, + 0x7b, 0x89, 0xa2, 0x0b, 0x70, 0x7d, 0x9c, 0x15, 0xf0, 0x41, 0x91, 0xe6, 0x0a, 0xcc, 0xb5, 0x7b, + 0x76, 0x68, 0xfb, 0x14, 0xe3, 0x04, 0x76, 0x95, 0x63, 0xa3, 0x78, 0x68, 0x50, 0xa1, 0x79, 0x05, + 0xa6, 0x6f, 0xbb, 0x94, 0x0b, 0x92, 0xdd, 0x6a, 0x8d, 0x47, 0x02, 0xfc, 0xee, 0x3e, 0x04, 0xe5, + 0x30, 0xd8, 0x13, 0x56, 0x2a, 0xc7, 0x55, 0xb0, 0x14, 0x06, 0x7b, 0xdc, 0x04, 0xf1, 0xb2, 0x72, + 0x10, 0x4a, 0xdd, 0xcc, 0x99, 0xf2, 0xcf, 0xf8, 0xae, 0x36, 0x50, 0x1e, 0x66, 0x60, 0xc8, 0x83, + 0x59, 0x98, 0x17, 0xa1, 0x14, 0x0a, 0xfa, 0x91, 0x45, 0xb6, 0xe4, 0x4a, 0xdc, 0x4a, 0x46, 0x54, + 0xc6, 0x77, 0x34, 0xd0, 0x5f, 0xf6, 0x7a, 0xe4, 0x8b, 0xd0, 0x61, 0x55, 0x5a, 0x3d, 0xaf, 0x4e, + 0xe9, 0xff, 0x30, 0x07, 0x35, 0xc9, 0xc6, 0x24, 0xde, 0x3f, 0x93, 0x95, 0x4d, 0xa8, 0xb2, 0x25, + 0x2d, 0x82, 0xdb, 0x51, 0x4e, 0xa2, 0xba, 0xba, 0xaa, 0xbc, 0xf5, 0x29, 0x36, 0x78, 0x79, 0x72, + 0x93, 0x13, 0x7d, 0xcd, 0xa7, 0x61, 0xdf, 0x84, 0x56, 0x0c, 0x68, 0xbe, 0x03, 0xb3, 0x43, 0xc3, + 0x4c, 0x37, 0x76, 0x71, 0x3f, 0x32, 0x6b, 0xbb, 0xb8, 0x8f, 0x9e, 0x4e, 0x16, 0x91, 0xb3, 0xdc, + 0xd7, 0x9d, 0xc0, 0x6f, 0x5f, 0x0f, 0x43, 0xbb, 0x2f, 0x8b, 0xcc, 0xcf, 0xe7, 0x9e, 0xd3, 0x8c, + 0x3f, 0xe5, 0x40, 0x7f, 0xad, 0x87, 0xc3, 0xfe, 0x51, 0x9a, 0x17, 0x04, 0xd3, 0xf8, 0x7e, 0x37, + 0x94, 0x0e, 0x9a, 0x7f, 0xef, 0xbf, 0xd1, 0x05, 0xc5, 0x8d, 0x56, 0xd8, 0xa5, 0xa2, 0xd2, 0x2e, + 0xa9, 0xae, 0x6c, 0xe9, 0x50, 0x57, 0xb6, 0x9c, 0x79, 0x65, 0x99, 0x76, 0x4b, 0x11, 0x4e, 0x74, + 0xc9, 0x52, 0xef, 0x90, 0xdc, 0x61, 0xdf, 0x21, 0xc6, 0x27, 0x1a, 0x54, 0xde, 0xc4, 0x2d, 0x1a, + 0x84, 0xcc, 0x5a, 0x28, 0x64, 0xaf, 0x8d, 0xf1, 0xd4, 0xcb, 0x0d, 0x3f, 0xf5, 0xae, 0x41, 0xd9, + 0x75, 0x2c, 0x9b, 0xa9, 0x0d, 0x3f, 0xbc, 0x51, 0x4f, 0x8c, 0x92, 0xeb, 0x70, 0xfd, 0x1a, 0x3f, + 0x37, 0xfd, 0x53, 0x0d, 0x74, 0xc1, 0x33, 0x11, 0x94, 0x2f, 0x24, 0x96, 0xd3, 0x54, 0xba, 0x2c, + 0x7f, 0xe2, 0x8d, 0xde, 0x9e, 0x1a, 0x2c, 0x7b, 0x1d, 0x80, 0xc9, 0x4e, 0x92, 0x8b, 0xab, 0xb0, + 0xa4, 0xe4, 0x56, 0x90, 0x73, 0x39, 0xde, 0x9e, 0x32, 0x2b, 0x8c, 0x8a, 0x4f, 0x71, 0xa3, 0x04, + 0x05, 0x4e, 0x6d, 0xfc, 0x5b, 0x83, 0xb9, 0x9b, 0xb6, 0xd7, 0x5a, 0x73, 0x09, 0xb5, 0xfd, 0xd6, + 0x04, 0xb1, 0xcf, 0xf3, 0x50, 0x0a, 0xba, 0x96, 0x87, 0xb7, 0xa9, 0x64, 0x69, 0x79, 0xc4, 0x8e, + 0x84, 0x18, 0xcc, 0x62, 0xd0, 0xbd, 0x83, 0xb7, 0x29, 0xfa, 0x32, 0x94, 0x83, 0xae, 0x15, 0xba, + 0xed, 0x1d, 0x2a, 0xa5, 0x3f, 0x06, 0x71, 0x29, 0xe8, 0x9a, 0x8c, 0x22, 0x91, 0x2b, 0x98, 0x3e, + 0x64, 0xae, 0xc0, 0xf8, 0xfb, 0xbe, 0xed, 0x4f, 0xa0, 0xda, 0xcf, 0x43, 0xd9, 0xf5, 0xa9, 0xe5, + 0xb8, 0x24, 0x12, 0xc1, 0x59, 0xb5, 0x0e, 0xf9, 0x94, 0xef, 0x80, 0x9f, 0xa9, 0x4f, 0xd9, 0xda, + 0xe8, 0x25, 0x80, 0x6d, 0x2f, 0xb0, 0x25, 0xb5, 0x90, 0xc1, 0x39, 0xf5, 0xad, 0x60, 0x68, 0x11, + 0x7d, 0x85, 0x13, 0xb1, 0x19, 0x06, 0x47, 0xfa, 0x57, 0x0d, 0x16, 0x36, 0x70, 0x48, 0x5c, 0x42, + 0xb1, 0x4f, 0x65, 0xde, 0x6e, 0xdd, 0xdf, 0x0e, 0xd2, 0x09, 0x52, 0x6d, 0x28, 0x41, 0xfa, 0xf9, + 0xa4, 0x0b, 0x53, 0x91, 0x80, 0x48, 0xd3, 0x47, 0x91, 0x40, 0x54, 0x8c, 0x10, 0x91, 0xd4, 0x4c, + 0xc6, 0x31, 0x49, 0x7e, 0x93, 0x01, 0xa5, 0xf1, 0x23, 0xd1, 0x18, 0xa0, 0xdc, 0xd4, 0x83, 0x2b, + 0xec, 0x22, 0x48, 0x03, 0x3e, 0x64, 0xce, 0x1f, 0x83, 0x21, 0xdb, 0x91, 0xd1, 0xae, 0xf0, 0x33, + 0x0d, 0x96, 0xb2, 0xb9, 0x9a, 0xc4, 0xf3, 0xbe, 0x04, 0x05, 0xd7, 0xdf, 0x0e, 0xa2, 0x34, 0xd2, + 0x45, 0xf5, 0x83, 0x5a, 0xb9, 0xae, 0x20, 0x34, 0xfe, 0xa5, 0x41, 0x9d, 0xdb, 0xea, 0x23, 0x38, + 0xfe, 0x0e, 0xee, 0x58, 0xc4, 0x7d, 0x17, 0x47, 0xc7, 0xdf, 0xc1, 0x9d, 0x4d, 0xf7, 0x5d, 0x9c, + 0xd2, 0x8c, 0x42, 0x5a, 0x33, 0xd2, 0x81, 0x76, 0x71, 0x44, 0x9a, 0xb0, 0x94, 0x4a, 0x13, 0x1a, + 0x1f, 0x6a, 0xd0, 0xbc, 0x85, 0xe9, 0xf0, 0x56, 0x8f, 0x4e, 0x29, 0x3e, 0xd2, 0xe0, 0x61, 0x25, + 0x43, 0x93, 0xe8, 0xc3, 0x0b, 0x69, 0x7d, 0x50, 0x07, 0x58, 0xfb, 0x96, 0x94, 0xaa, 0x70, 0x15, + 0xf4, 0xb5, 0x5e, 0xa7, 0x13, 0x3f, 0x7c, 0x96, 0x41, 0x0f, 0xc5, 0xa7, 0x88, 0x3f, 0x84, 0xbb, + 0xac, 0x4a, 0x18, 0x8b, 0x32, 0x8c, 0x4b, 0x50, 0x93, 0x24, 0x92, 0xeb, 0x26, 0x94, 0x43, 0xf9, + 0x2d, 0xf1, 0xe3, 0x7f, 0x63, 0x01, 0xe6, 0x4c, 0xdc, 0x66, 0x9a, 0x18, 0xde, 0x71, 0xfd, 0x5d, + 0xb9, 0x8c, 0xf1, 0x9e, 0x06, 0xf3, 0x69, 0xb8, 0x9c, 0xeb, 0xff, 0xa1, 0x64, 0x3b, 0x4e, 0x88, + 0x09, 0x19, 0x79, 0x2c, 0xd7, 0x05, 0x8e, 0x19, 0x21, 0x27, 0x24, 0x97, 0x1b, 0x5b, 0x72, 0x86, + 0x05, 0xa7, 0x6e, 0x61, 0x7a, 0x17, 0xd3, 0x70, 0xa2, 0x3a, 0x70, 0x83, 0x45, 0x06, 0x9c, 0x58, + 0xaa, 0x45, 0xf4, 0x6b, 0x7c, 0xa0, 0x01, 0x4a, 0xae, 0x30, 0xc9, 0x31, 0x27, 0xa5, 0x9c, 0x4b, + 0x4b, 0x59, 0xb4, 0xca, 0x74, 0xba, 0x81, 0x8f, 0x7d, 0x9a, 0x7c, 0x62, 0xd6, 0x62, 0x28, 0x53, + 0xbf, 0x8b, 0xcb, 0x50, 0x8e, 0x4a, 0x97, 0xa8, 0x04, 0xf9, 0xeb, 0x9e, 0x57, 0x9f, 0x42, 0x3a, + 0x94, 0xd7, 0x65, 0x7d, 0xae, 0xae, 0x5d, 0xfc, 0x2a, 0xcc, 0x0e, 0x45, 0xfe, 0xa8, 0x0c, 0xd3, + 0xaf, 0x06, 0x3e, 0xae, 0x4f, 0xa1, 0x3a, 0xe8, 0x37, 0x5c, 0xdf, 0x0e, 0xfb, 0xc2, 0xd3, 0xd6, + 0x1d, 0x34, 0x0b, 0x55, 0xee, 0x71, 0x24, 0x00, 0xaf, 0x7e, 0xbb, 0x01, 0xb5, 0xbb, 0x7c, 0x33, + 0x9b, 0x38, 0xbc, 0xe7, 0xb6, 0x30, 0xb2, 0xa0, 0x3e, 0xdc, 0x7c, 0x8c, 0x9e, 0x50, 0xea, 0x68, + 0x46, 0x8f, 0x72, 0x73, 0x94, 0x78, 0x8c, 0x29, 0xf4, 0x36, 0xcc, 0xa4, 0xdb, 0x82, 0x91, 0xda, + 0x24, 0x2a, 0x7b, 0x87, 0x0f, 0x9a, 0xdc, 0x82, 0x5a, 0xaa, 0xcb, 0x17, 0x5d, 0x50, 0xce, 0xad, + 0xea, 0x04, 0x6e, 0xaa, 0x5f, 0x29, 0xc9, 0x4e, 0x5c, 0xc1, 0x7d, 0xba, 0x17, 0x31, 0x83, 0x7b, + 0x65, 0xc3, 0xe2, 0x41, 0xdc, 0xdb, 0x70, 0x6a, 0x5f, 0x6b, 0x21, 0x7a, 0x52, 0x39, 0x7f, 0x56, + 0x0b, 0xe2, 0x41, 0x4b, 0xec, 0x01, 0xda, 0xdf, 0xcd, 0x8a, 0x2e, 0xab, 0x4f, 0x20, 0xab, 0x97, + 0xb7, 0x79, 0x65, 0x6c, 0xfc, 0x58, 0x70, 0xdf, 0xd3, 0xe0, 0x74, 0x46, 0x3f, 0x20, 0xba, 0xa6, + 0x9c, 0x6e, 0x74, 0x53, 0x63, 0xf3, 0xe9, 0xc3, 0x11, 0xc5, 0x8c, 0xf8, 0x30, 0x3b, 0xd4, 0x22, + 0x87, 0x2e, 0x65, 0xb6, 0x0d, 0xec, 0xef, 0x15, 0x6c, 0x3e, 0x31, 0x1e, 0x72, 0xbc, 0x1e, 0x8b, + 0x85, 0xd3, 0x7d, 0x65, 0x19, 0xeb, 0xa9, 0xbb, 0xcf, 0x0e, 0x3a, 0xd0, 0xb7, 0xa0, 0x96, 0x6a, + 0x00, 0xcb, 0xd0, 0x78, 0x55, 0x93, 0xd8, 0x41, 0x53, 0xbf, 0x03, 0x7a, 0xb2, 0x4f, 0x0b, 0xad, + 0x64, 0xdd, 0xa5, 0x7d, 0x13, 0x1f, 0xe6, 0x2a, 0x0d, 0xda, 0x30, 0x46, 0x5c, 0xa5, 0x7d, 0x9d, + 0x2b, 0xe3, 0x5f, 0xa5, 0xc4, 0xfc, 0x23, 0xaf, 0xd2, 0xa1, 0x97, 0x78, 0x4f, 0x83, 0x45, 0x75, + 0x9b, 0x0f, 0x5a, 0xcd, 0xd2, 0xcd, 0xec, 0x86, 0xa6, 0xe6, 0xb5, 0x43, 0xd1, 0xc4, 0x52, 0xdc, + 0x85, 0x99, 0x74, 0x33, 0x4b, 0x86, 0x14, 0x95, 0xfd, 0x3f, 0xcd, 0x4b, 0x63, 0xe1, 0xc6, 0x8b, + 0xbd, 0x01, 0xd5, 0x44, 0x41, 0x1f, 0x3d, 0x3e, 0x42, 0x8f, 0x93, 0xe5, 0xa0, 0x83, 0x24, 0xb9, + 0x03, 0xb5, 0x54, 0x11, 0x37, 0x4b, 0x87, 0x15, 0xb5, 0xf5, 0xe6, 0xc5, 0x71, 0x50, 0xe3, 0x0d, + 0xec, 0x40, 0x2d, 0x55, 0x52, 0xcb, 0x58, 0x49, 0x55, 0x41, 0xcc, 0x58, 0x49, 0x59, 0xa1, 0x33, + 0xa6, 0xd0, 0xb7, 0x12, 0xd5, 0xbb, 0x54, 0x85, 0x14, 0x5d, 0x1d, 0x39, 0x8f, 0xaa, 0x40, 0xdc, + 0x5c, 0x3d, 0x0c, 0x49, 0xcc, 0xc2, 0x6b, 0x50, 0x89, 0x0b, 0x73, 0xe8, 0x7c, 0xa6, 0x59, 0x38, + 0xcc, 0x49, 0x6d, 0x42, 0x51, 0x14, 0xc9, 0x90, 0x91, 0x51, 0x0e, 0x4f, 0x54, 0xd0, 0x9a, 0x8f, + 0x2a, 0x71, 0xd2, 0xf5, 0x23, 0x63, 0x0a, 0x99, 0x50, 0x14, 0x69, 0xd8, 0x8c, 0x49, 0x53, 0xa5, + 0x84, 0xe6, 0x68, 0x1c, 0x91, 0xbb, 0x9d, 0x42, 0x1b, 0x50, 0xe0, 0xe9, 0x4a, 0xb4, 0x3c, 0x2a, + 0x95, 0x39, 0x6a, 0xc6, 0x54, 0xb6, 0xd3, 0x98, 0x42, 0x5f, 0x87, 0x02, 0x7f, 0x95, 0x67, 0xcc, + 0x98, 0xcc, 0x47, 0x36, 0x47, 0xa2, 0x44, 0x2c, 0x3a, 0xa0, 0x27, 0xb3, 0x15, 0x19, 0xe6, 0x55, + 0x91, 0xcf, 0x69, 0x8e, 0x83, 0x19, 0xad, 0xf2, 0x7d, 0x0d, 0x1a, 0x59, 0x81, 0x2d, 0xca, 0xf4, + 0xa1, 0xa3, 0xa2, 0xf3, 0xe6, 0x33, 0x87, 0xa4, 0x8a, 0x45, 0xf8, 0x2e, 0xcc, 0x29, 0xc2, 0x29, + 0x74, 0x25, 0x6b, 0xbe, 0x8c, 0x48, 0xb0, 0xf9, 0xd4, 0xf8, 0x04, 0xf1, 0xda, 0x1b, 0x50, 0xe0, + 0x61, 0x50, 0xc6, 0xf1, 0x25, 0xa3, 0xaa, 0x0c, 0x85, 0x48, 0x45, 0x51, 0xc6, 0x14, 0xc2, 0xa0, + 0x27, 0x63, 0xa2, 0x8c, 0xf3, 0x53, 0x84, 0x53, 0xcd, 0x0b, 0x63, 0x60, 0xc6, 0xcb, 0x58, 0x00, + 0x83, 0x98, 0x04, 0x3d, 0x96, 0xb5, 0xf5, 0x74, 0x58, 0xd4, 0x7c, 0xfc, 0x40, 0xbc, 0x68, 0x81, + 0xd5, 0x1e, 0xe8, 0x1b, 0x61, 0x70, 0xbf, 0x1f, 0x45, 0x00, 0xff, 0x9b, 0x7d, 0xdd, 0x78, 0xe6, + 0x1b, 0xd7, 0xda, 0x2e, 0xdd, 0xe9, 0x6d, 0x31, 0x23, 0x73, 0x45, 0xe0, 0x3e, 0xe9, 0x06, 0xf2, + 0xeb, 0x8a, 0xeb, 0x53, 0x1c, 0xfa, 0xb6, 0x77, 0x85, 0xcf, 0x25, 0xa1, 0xdd, 0xad, 0xad, 0x22, + 0xff, 0xbf, 0xf6, 0xdf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x6a, 0x17, 0x33, 0xc5, 0x65, 0x39, 0x00, + 0x00, } // Reference imports to suppress errors if they are not otherwise used. @@ -4238,6 +4240,8 @@ type MilvusServiceClient interface { Dummy(ctx context.Context, in *DummyRequest, opts ...grpc.CallOption) (*DummyResponse, error) // TODO: remove RegisterLink(ctx context.Context, in *RegisterLinkRequest, opts ...grpc.CallOption) (*RegisterLinkResponse, error) + // https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy + GetMetrics(ctx context.Context, in *GetMetricsRequest, opts ...grpc.CallOption) (*GetMetricsResponse, error) } type milvusServiceClient struct { @@ -4509,6 +4513,15 @@ func (c *milvusServiceClient) RegisterLink(ctx context.Context, in *RegisterLink return out, nil } +func (c *milvusServiceClient) GetMetrics(ctx context.Context, in *GetMetricsRequest, opts ...grpc.CallOption) (*GetMetricsResponse, error) { + out := new(GetMetricsResponse) + err := c.cc.Invoke(ctx, "/milvus.proto.milvus.MilvusService/GetMetrics", in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // MilvusServiceServer is the server API for MilvusService service. type MilvusServiceServer interface { CreateCollection(context.Context, *CreateCollectionRequest) (*commonpb.Status, error) @@ -4541,6 +4554,8 @@ type MilvusServiceServer interface { Dummy(context.Context, *DummyRequest) (*DummyResponse, error) // TODO: remove RegisterLink(context.Context, *RegisterLinkRequest) (*RegisterLinkResponse, error) + // https://wiki.lfaidata.foundation/display/MIL/MEP+8+--+Add+metrics+for+proxy + GetMetrics(context.Context, *GetMetricsRequest) (*GetMetricsResponse, error) } // UnimplementedMilvusServiceServer can be embedded to have forward compatible implementations. @@ -4634,6 +4649,9 @@ func (*UnimplementedMilvusServiceServer) Dummy(ctx context.Context, req *DummyRe func (*UnimplementedMilvusServiceServer) RegisterLink(ctx context.Context, req *RegisterLinkRequest) (*RegisterLinkResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method RegisterLink not implemented") } +func (*UnimplementedMilvusServiceServer) GetMetrics(ctx context.Context, req *GetMetricsRequest) (*GetMetricsResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method GetMetrics not implemented") +} func RegisterMilvusServiceServer(s *grpc.Server, srv MilvusServiceServer) { s.RegisterService(&_MilvusService_serviceDesc, srv) @@ -5161,6 +5179,24 @@ func _MilvusService_RegisterLink_Handler(srv interface{}, ctx context.Context, d return interceptor(ctx, in, info, handler) } +func _MilvusService_GetMetrics_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(GetMetricsRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(MilvusServiceServer).GetMetrics(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: "/milvus.proto.milvus.MilvusService/GetMetrics", + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(MilvusServiceServer).GetMetrics(ctx, req.(*GetMetricsRequest)) + } + return interceptor(ctx, in, info, handler) +} + var _MilvusService_serviceDesc = grpc.ServiceDesc{ ServiceName: "milvus.proto.milvus.MilvusService", HandlerType: (*MilvusServiceServer)(nil), @@ -5281,6 +5317,10 @@ var _MilvusService_serviceDesc = grpc.ServiceDesc{ MethodName: "RegisterLink", Handler: _MilvusService_RegisterLink_Handler, }, + { + MethodName: "GetMetrics", + Handler: _MilvusService_GetMetrics_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "milvus.proto", diff --git a/internal/proxy/error.go b/internal/proxy/error.go index a15c786569..19fa8d049f 100644 --- a/internal/proxy/error.go +++ b/internal/proxy/error.go @@ -1,3 +1,14 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + package proxy import ( @@ -46,3 +57,11 @@ func errDimLessThanOrEqualToZero(dim int) error { func errDimShouldDivide8(dim int) error { return fmt.Errorf("dim(%d) should divide 8", dim) } + +func msgProxyIsUnhealthy(id UniqueID) string { + return fmt.Sprintf("proxy %d is unhealthy", id) +} + +func errProxyIsUnhealthy(id UniqueID) error { + return errors.New(msgProxyIsUnhealthy(id)) +} diff --git a/internal/proxy/error_test.go b/internal/proxy/error_test.go new file mode 100644 index 0000000000..626dae926c --- /dev/null +++ b/internal/proxy/error_test.go @@ -0,0 +1,41 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +package proxy + +import ( + "testing" + + "github.com/milvus-io/milvus/internal/log" + "go.uber.org/zap" +) + +func TestMsgProxyIsUnhealthy(t *testing.T) { + ids := []UniqueID{ + 1, + } + + for _, id := range ids { + log.Info("TestMsgProxyIsUnhealthy", + zap.String("msg", msgProxyIsUnhealthy(id))) + } +} + +func TestErrProxyIsUnhealthy(t *testing.T) { + ids := []UniqueID{ + 1, + } + + for _, id := range ids { + log.Info("TestErrProxyIsUnhealthy", + zap.Error(errProxyIsUnhealthy(id))) + } +} diff --git a/internal/proxy/impl.go b/internal/proxy/impl.go index 906316d538..947005816c 100644 --- a/internal/proxy/impl.go +++ b/internal/proxy/impl.go @@ -18,6 +18,8 @@ import ( "os" "strconv" + "github.com/milvus-io/milvus/internal/util/metricsinfo" + "go.uber.org/zap" "github.com/milvus-io/milvus/internal/log" @@ -1972,6 +1974,73 @@ func (node *Proxy) RegisterLink(ctx context.Context, req *milvuspb.RegisterLinkR }, nil } +// TODO(dragondriver): cache the Metrics and set a retention to the cache +func (node *Proxy) GetMetrics(ctx context.Context, req *milvuspb.GetMetricsRequest) (*milvuspb.GetMetricsResponse, error) { + log.Debug("Proxy.GetMetrics", + zap.Int64("node_id", Params.ProxyID), + zap.String("req", req.Request)) + + if !node.checkHealthy() { + log.Warn("Proxy.GetMetrics failed", + zap.Int64("node_id", Params.ProxyID), + zap.String("req", req.Request), + zap.Error(errProxyIsUnhealthy(Params.ProxyID))) + + return &milvuspb.GetMetricsResponse{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: msgProxyIsUnhealthy(Params.ProxyID), + }, + Response: "", + }, nil + } + + metricType, err := metricsinfo.ParseMetricType(req.Request) + if err != nil { + log.Warn("Proxy.GetMetrics failed to parse metric type", + zap.Int64("node_id", Params.ProxyID), + zap.String("req", req.Request), + zap.Error(err)) + + return &milvuspb.GetMetricsResponse{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: err.Error(), + }, + Response: "", + }, nil + } + + log.Debug("Proxy.GetMetrics", + zap.String("metric_type", metricType)) + + if metricType == metricsinfo.SystemInfoMetrics { + metrics, err := getSystemInfoMetrics(ctx, req, node) + + log.Debug("Proxy.GetMetrics", + zap.Int64("node_id", Params.ProxyID), + zap.String("req", req.Request), + zap.String("metric_type", metricType), + zap.Any("metrics", metrics), // TODO(dragondriver): necessary? may be very large + zap.Error(err)) + + return metrics, err + } + + log.Debug("Proxy.GetMetrics failed, request metric type is not implemented yet", + zap.Int64("node_id", Params.ProxyID), + zap.String("req", req.Request), + zap.String("metric_type", metricType)) + + return &milvuspb.GetMetricsResponse{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: metricsinfo.MsgUnimplementedMetric, + }, + Response: "", + }, nil +} + // checkHealthy checks proxy state is Healthy func (node *Proxy) checkHealthy() bool { code := node.stateCode.Load().(internalpb.StateCode) diff --git a/internal/proxy/metrics_info.go b/internal/proxy/metrics_info.go new file mode 100644 index 0000000000..8a508ee64c --- /dev/null +++ b/internal/proxy/metrics_info.go @@ -0,0 +1,128 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +package proxy + +import ( + "context" + + "github.com/milvus-io/milvus/internal/proto/commonpb" + + "github.com/milvus-io/milvus/internal/util/typeutil" + + "github.com/milvus-io/milvus/internal/util/metricsinfo" + + "github.com/milvus-io/milvus/internal/proto/milvuspb" +) + +func getSystemInfoMetrics( + ctx context.Context, + request *milvuspb.GetMetricsRequest, + node *Proxy, +) (*milvuspb.GetMetricsResponse, error) { + + var err error + + systemTopology := metricsinfo.SystemTopology{ + NodesInfo: make([]metricsinfo.SystemTopologyNode, 0), + } + + identifierMap := make(map[string]int) + + proxyRoleName := metricsinfo.ConstructComponentName(typeutil.ProxyRole, Params.ProxyID) + identifierMap[proxyRoleName] = getUniqueIntGeneratorIns().get() + proxyTopologyNode := metricsinfo.SystemTopologyNode{ + Identifier: identifierMap[proxyRoleName], + Connected: make([]metricsinfo.ConnectionEdge, 0), + Infos: &metricsinfo.ProxyInfos{ + BaseComponentInfos: metricsinfo.BaseComponentInfos{ + HasError: false, + ErrorReason: "", + Name: proxyRoleName, + }, + }, + } + + queryCoordResp, queryCoordErr := node.queryCoord.GetMetrics(ctx, request) + skipQueryCoord := false + queryCoordRoleName := "" + if queryCoordErr != nil || queryCoordResp == nil { + skipQueryCoord = true + } else { + queryCoordRoleName = queryCoordResp.ComponentName + identifierMap[queryCoordRoleName] = getUniqueIntGeneratorIns().get() + } + + if !skipQueryCoord { + proxyTopologyNode.Connected = append(proxyTopologyNode.Connected, metricsinfo.ConnectionEdge{ + ConnectedIdentifier: identifierMap[queryCoordRoleName], + Type: metricsinfo.Forward, + TargetType: typeutil.QueryCoordRole, + }) + + queryCoordTopology := metricsinfo.QueryCoordTopology{} + err = metricsinfo.UnmarshalTopology(queryCoordResp.Response, &queryCoordTopology) + if err == nil { + // query coord in system topology graph + queryCoordTopologyNode := metricsinfo.SystemTopologyNode{ + Identifier: identifierMap[queryCoordRoleName], + Connected: make([]metricsinfo.ConnectionEdge, 0), + Infos: &queryCoordTopology.Cluster.Self, + } + + // add query nodes to system topology graph + for _, queryNode := range queryCoordTopology.Cluster.ConnectedNodes { + identifier := getUniqueIntGeneratorIns().get() + identifierMap[queryNode.Name] = identifier + queryNodeTopologyNode := metricsinfo.SystemTopologyNode{ + Identifier: identifier, + Connected: nil, + Infos: &queryNode, + } + systemTopology.NodesInfo = append(systemTopology.NodesInfo, queryNodeTopologyNode) + queryCoordTopologyNode.Connected = append(queryCoordTopologyNode.Connected, metricsinfo.ConnectionEdge{ + ConnectedIdentifier: identifier, + Type: metricsinfo.CoordConnectToNode, + TargetType: typeutil.QueryNodeRole, + }) + } + + // add query coord to system topology graph + systemTopology.NodesInfo = append(systemTopology.NodesInfo, queryCoordTopologyNode) + } + } + + // TODO(dragondriver): integrate other coordinator + + // add proxy to system topology graph + systemTopology.NodesInfo = append(systemTopology.NodesInfo, proxyTopologyNode) + + resp, err := metricsinfo.MarshalTopology(systemTopology) + if err != nil { + return &milvuspb.GetMetricsResponse{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_UnexpectedError, + Reason: err.Error(), + }, + Response: "", + ComponentName: metricsinfo.ConstructComponentName(typeutil.ProxyRole, Params.ProxyID), + }, nil + } + + return &milvuspb.GetMetricsResponse{ + Status: &commonpb.Status{ + ErrorCode: commonpb.ErrorCode_Success, + Reason: "", + }, + Response: resp, + ComponentName: metricsinfo.ConstructComponentName(typeutil.ProxyRole, Params.ProxyID), + }, nil +} diff --git a/internal/querycoord/metrics_info.go b/internal/querycoord/metrics_info.go index b27ccf6677..4fc0ec9329 100644 --- a/internal/querycoord/metrics_info.go +++ b/internal/querycoord/metrics_info.go @@ -14,6 +14,8 @@ package querycoord import ( "context" + "github.com/milvus-io/milvus/internal/util/typeutil" + "go.uber.org/zap" "github.com/milvus-io/milvus/internal/log" @@ -21,7 +23,6 @@ import ( "github.com/milvus-io/milvus/internal/proto/commonpb" "github.com/milvus-io/milvus/internal/proto/milvuspb" "github.com/milvus-io/milvus/internal/util/metricsinfo" - "github.com/milvus-io/milvus/internal/util/typeutil" ) // TODO(dragondriver): add more detail metrics @@ -31,11 +32,13 @@ func getSystemInfoMetrics( qc *QueryCoord, ) (*milvuspb.GetMetricsResponse, error) { - coordTopology := metricsinfo.CoordTopology{ - Self: metricsinfo.ComponentInfos{ - Name: metricsinfo.ConstructComponentName(typeutil.QueryCoordRole, Params.QueryCoordID), + clusterTopology := metricsinfo.QueryClusterTopology{ + Self: metricsinfo.QueryCoordInfos{ + BaseComponentInfos: metricsinfo.BaseComponentInfos{ + Name: metricsinfo.ConstructComponentName(typeutil.QueryCoordRole, Params.QueryCoordID), + }, }, - ConnectedNodes: make([]metricsinfo.ComponentInfos, 0), + ConnectedNodes: make([]metricsinfo.QueryNodeInfos, 0), } nodesMetrics := qc.cluster.getMetrics(ctx, req) @@ -43,11 +46,13 @@ func getSystemInfoMetrics( if nodeMetrics.err != nil { log.Warn("invalid metrics of query node was found", zap.Error(nodeMetrics.err)) - coordTopology.ConnectedNodes = append(coordTopology.ConnectedNodes, metricsinfo.ComponentInfos{ - HasError: true, - ErrorReason: nodeMetrics.err.Error(), - // Name doesn't matter here cause we can't get it when error occurs, using address as the Name? - Name: "", + clusterTopology.ConnectedNodes = append(clusterTopology.ConnectedNodes, metricsinfo.QueryNodeInfos{ + BaseComponentInfos: metricsinfo.BaseComponentInfos{ + HasError: true, + ErrorReason: nodeMetrics.err.Error(), + // Name doesn't matter here cause we can't get it when error occurs, using address as the Name? + Name: "", + }, }) continue } @@ -56,37 +61,50 @@ func getSystemInfoMetrics( log.Warn("invalid metrics of query node was found", zap.Any("error_code", nodeMetrics.resp.Status.ErrorCode), zap.Any("error_reason", nodeMetrics.resp.Status.Reason)) - coordTopology.ConnectedNodes = append(coordTopology.ConnectedNodes, metricsinfo.ComponentInfos{ - HasError: true, - ErrorReason: nodeMetrics.resp.Status.Reason, - Name: nodeMetrics.resp.ComponentName, + clusterTopology.ConnectedNodes = append(clusterTopology.ConnectedNodes, metricsinfo.QueryNodeInfos{ + BaseComponentInfos: metricsinfo.BaseComponentInfos{ + HasError: true, + ErrorReason: nodeMetrics.resp.Status.Reason, + Name: nodeMetrics.resp.ComponentName, + }, }) continue } - infos := metricsinfo.ComponentInfos{} - err := infos.Unmarshal(nodeMetrics.resp.Response) + infos := metricsinfo.QueryNodeInfos{} + err := metricsinfo.UnmarshalComponentInfos(nodeMetrics.resp.Response, &infos) if err != nil { log.Warn("invalid metrics of query node was found", zap.Error(err)) - coordTopology.ConnectedNodes = append(coordTopology.ConnectedNodes, metricsinfo.ComponentInfos{ - HasError: true, - ErrorReason: err.Error(), - Name: nodeMetrics.resp.ComponentName, + clusterTopology.ConnectedNodes = append(clusterTopology.ConnectedNodes, metricsinfo.QueryNodeInfos{ + BaseComponentInfos: metricsinfo.BaseComponentInfos{ + HasError: true, + ErrorReason: err.Error(), + Name: nodeMetrics.resp.ComponentName, + }, }) continue } - coordTopology.ConnectedNodes = append(coordTopology.ConnectedNodes, infos) + clusterTopology.ConnectedNodes = append(clusterTopology.ConnectedNodes, infos) } - resp, err := coordTopology.Marshal() + coordTopology := metricsinfo.QueryCoordTopology{ + Cluster: clusterTopology, + Connections: metricsinfo.ConnTopology{ + Name: metricsinfo.ConstructComponentName(typeutil.QueryCoordRole, Params.QueryCoordID), + // TODO(dragondriver): connection info + }, + } + + resp, err := metricsinfo.MarshalTopology(coordTopology) if err != nil { return &milvuspb.GetMetricsResponse{ Status: &commonpb.Status{ ErrorCode: commonpb.ErrorCode_UnexpectedError, Reason: err.Error(), }, - Response: "", + Response: "", + ComponentName: metricsinfo.ConstructComponentName(typeutil.QueryCoordRole, Params.QueryCoordID), }, nil } @@ -95,6 +113,7 @@ func getSystemInfoMetrics( ErrorCode: commonpb.ErrorCode_Success, Reason: "", }, - Response: resp, + Response: resp, + ComponentName: metricsinfo.ConstructComponentName(typeutil.QueryCoordRole, Params.QueryCoordID), }, nil } diff --git a/internal/querynode/metrics_info.go b/internal/querynode/metrics_info.go index 15965ae628..fb8bfd3890 100644 --- a/internal/querynode/metrics_info.go +++ b/internal/querynode/metrics_info.go @@ -22,10 +22,12 @@ import ( func getSystemInfoMetrics(ctx context.Context, req *milvuspb.GetMetricsRequest, node *QueryNode) (*milvuspb.GetMetricsResponse, error) { // TODO(dragondriver): add more metrics - nodeInfos := metricsinfo.ComponentInfos{ - Name: metricsinfo.ConstructComponentName(typeutil.QueryNodeRole, Params.QueryNodeID), + nodeInfos := metricsinfo.QueryNodeInfos{ + BaseComponentInfos: metricsinfo.BaseComponentInfos{ + Name: metricsinfo.ConstructComponentName(typeutil.QueryNodeRole, Params.QueryNodeID), + }, } - resp, err := nodeInfos.Marshal() + resp, err := metricsinfo.MarshalComponentInfos(nodeInfos) if err != nil { return &milvuspb.GetMetricsResponse{ Status: &commonpb.Status{ diff --git a/internal/util/metricsinfo/metrics_info.go b/internal/util/metricsinfo/metrics_info.go new file mode 100644 index 0000000000..e044eed70b --- /dev/null +++ b/internal/util/metricsinfo/metrics_info.go @@ -0,0 +1,57 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +package metricsinfo + +import ( + "encoding/json" +) + +// ComponentInfos defines the interface of all component infos +type ComponentInfos interface { +} + +// MarshalComponentInfos returns the json string of ComponentInfos +func MarshalComponentInfos(infos ComponentInfos) (string, error) { + binary, err := json.Marshal(infos) + return string(binary), err +} + +// UnmarshalComponentInfos constructs a ComponentInfos object using a json string +func UnmarshalComponentInfos(s string, infos ComponentInfos) error { + return json.Unmarshal([]byte(s), infos) +} + +// BaseComponentInfos contains basic information that all components should have. +type BaseComponentInfos struct { + HasError bool `json:"has_error"` + ErrorReason string `json:"error_reason"` + Name string `json:"name"` + // TODO(dragondriver): more required information +} + +// QueryNodeInfos implements ComponentInfos +type QueryNodeInfos struct { + BaseComponentInfos + // TODO(dragondriver): add more detail metrics +} + +// QueryCoordInfos implements ComponentInfos +type QueryCoordInfos struct { + BaseComponentInfos + // TODO(dragondriver): add more detail metrics +} + +// ProxyInfos implements ComponentInfos +type ProxyInfos struct { + BaseComponentInfos + // TODO(dragondriver): add more detail metrics +} diff --git a/internal/util/metricsinfo/metrics_info_test.go b/internal/util/metricsinfo/metrics_info_test.go new file mode 100644 index 0000000000..4c13795d7c --- /dev/null +++ b/internal/util/metricsinfo/metrics_info_test.go @@ -0,0 +1,69 @@ +// Copyright (C) 2019-2020 Zilliz. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software distributed under the License +// is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +// or implied. See the License for the specific language governing permissions and limitations under the License. + +package metricsinfo + +import ( + "testing" + + "go.uber.org/zap" + + "github.com/milvus-io/milvus/internal/log" + + "github.com/milvus-io/milvus/internal/util/typeutil" + "github.com/stretchr/testify/assert" +) + +func TestBaseComponentInfos_Codec(t *testing.T) { + infos1 := BaseComponentInfos{ + Name: ConstructComponentName(typeutil.ProxyRole, 1), + } + s, err := MarshalComponentInfos(infos1) + assert.Equal(t, nil, err) + log.Info("TestBaseComponentInfos_Codec", + zap.String("marshaled_result", s)) + var infos2 BaseComponentInfos + err = UnmarshalComponentInfos(s, &infos2) + assert.Equal(t, nil, err) + assert.Equal(t, infos1.Name, infos2.Name) +} + +func TestQueryNodeInfos_Codec(t *testing.T) { + infos1 := QueryNodeInfos{ + BaseComponentInfos: BaseComponentInfos{ + Name: ConstructComponentName(typeutil.QueryNodeRole, 1), + }, + } + s, err := MarshalComponentInfos(infos1) + assert.Equal(t, nil, err) + log.Info("TestQueryNodeInfos_Codec", + zap.String("marshaled_result", s)) + var infos2 QueryNodeInfos + err = UnmarshalComponentInfos(s, &infos2) + assert.Equal(t, nil, err) + assert.Equal(t, infos1.Name, infos2.Name) +} + +func TestQueryCoordInfos_Codec(t *testing.T) { + infos1 := QueryCoordInfos{ + BaseComponentInfos: BaseComponentInfos{ + Name: ConstructComponentName(typeutil.QueryCoordRole, 1), + }, + } + s, err := MarshalComponentInfos(infos1) + assert.Equal(t, nil, err) + log.Info("TestQueryCoordInfos_Codec", + zap.String("marshaled_result", s)) + var infos2 QueryCoordInfos + err = UnmarshalComponentInfos(s, &infos2) + assert.Equal(t, nil, err) + assert.Equal(t, infos1.Name, infos2.Name) +} diff --git a/internal/util/metricsinfo/topology.go b/internal/util/metricsinfo/topology.go index 2280774d8d..abb54ce697 100644 --- a/internal/util/metricsinfo/topology.go +++ b/internal/util/metricsinfo/topology.go @@ -26,46 +26,27 @@ func ConstructComponentName(role string, id typeutil.UniqueID) string { return role + strconv.Itoa(int(id)) } -// TODO(dragondriver): how different components extend their metrics information? -// so maybe providing a interface instead of a struct is better? -// then every components can implement it to extend metrics. - -// ComponentInfos shows the all necessary metrics of node. -type ComponentInfos struct { - HasError bool `json:"has_error"` - ErrorReason string `json:"error_reason"` - Name string `json:"name"` - // TODO(dragondriver): more required information +// Topology defines the interface of topology graph between different components +type Topology interface { } -// Marshal returns the json string of ComponentInfos -func (infos *ComponentInfos) Marshal() (string, error) { - binary, err := json.Marshal(infos) - return string(binary), err -} - -// Unmarshal constructs a ComponentInfos object using a json string -func (infos *ComponentInfos) Unmarshal(s string) error { - return json.Unmarshal([]byte(s), infos) -} - -// CoordTopology used for coordinator to show their managed nodes. -type CoordTopology struct { - Self ComponentInfos `json:"self"` - ConnectedNodes []ComponentInfos `json:"connected_nodes"` -} - -// Marshal returns the json string of CoordTopology -func (topology *CoordTopology) Marshal() (string, error) { +// MarshalTopology returns the json string of Topology +func MarshalTopology(topology Topology) (string, error) { binary, err := json.Marshal(topology) return string(binary), err } -// Unmarshal constructs a CoordTopology object using a json string -func (topology *CoordTopology) Unmarshal(s string) error { +// UnmarshalTopology constructs a Topology object using the json string +func UnmarshalTopology(s string, topology Topology) error { return json.Unmarshal([]byte(s), topology) } +// QueryClusterTopology shows the topology between QueryCoord and QueryNodes +type QueryClusterTopology struct { + Self QueryCoordInfos `json:"self"` + ConnectedNodes []QueryNodeInfos `json:"connected_nodes"` +} + // TODO(dragondriver) // necessary to show all connection edge in topology graph? // for example, in system, Proxy connects to RootCoord and RootCoord also connects to Proxy, @@ -76,28 +57,34 @@ type ConnTopology struct { ConnectedComponents []string `json:"connected_components"` } -// Marshal returns the json string of ConnTopology -func (topology *ConnTopology) Marshal() (string, error) { - binary, err := json.Marshal(topology) - return string(binary), err +// QueryCoordTopology shows the whole metrics of query cluster +type QueryCoordTopology struct { + Cluster QueryClusterTopology `json:"cluster"` + Connections ConnTopology `json:"connections"` } -// Unmarshal constructs a ConnTopology object using a json string -func (topology *ConnTopology) Unmarshal(s string) error { - return json.Unmarshal([]byte(s), topology) +type ConnectionType string + +const ( + CoordConnectToNode ConnectionType = "manage" + Forward ConnectionType = "forward" +) + +type ConnectionTargetType string + +type ConnectionEdge struct { + ConnectedIdentifier int `json:"connected_identifier"` + Type ConnectionType `json:"type"` + TargetType ConnectionTargetType `json:"target_type"` // RootCoord, DataCoord ... +} + +type SystemTopologyNode struct { + Identifier int `json:"identifier"` // unique in the SystemTopology graph + Connected []ConnectionEdge `json:"connected"` + Infos ComponentInfos `json:"infos"` } // SystemTopology shows the system topology type SystemTopology struct { -} - -// Marshal returns the json string of SystemTopology -func (topology *SystemTopology) Marshal() (string, error) { - binary, err := json.Marshal(topology) - return string(binary), err -} - -// Unmarshal constructs a SystemTopology object using a json string -func (topology *SystemTopology) Unmarshal(s string) error { - return json.Unmarshal([]byte(s), topology) + NodesInfo []SystemTopologyNode `json:"nodes_info"` } diff --git a/internal/util/metricsinfo/topology_test.go b/internal/util/metricsinfo/topology_test.go index fa11269030..5b26459a2e 100644 --- a/internal/util/metricsinfo/topology_test.go +++ b/internal/util/metricsinfo/topology_test.go @@ -38,38 +38,34 @@ func TestConstructComponentName(t *testing.T) { } } -func TestComponentInfos_Codec(t *testing.T) { - infos1 := ComponentInfos{ - Name: ConstructComponentName(typeutil.ProxyRole, 1), - } - s, err := infos1.Marshal() - assert.Equal(t, nil, err) - var infos2 ComponentInfos - err = infos2.Unmarshal(s) - assert.Equal(t, nil, err) - assert.Equal(t, infos1.Name, infos2.Name) -} - -func TestCoordTopology_Codec(t *testing.T) { - topology1 := CoordTopology{ - Self: ComponentInfos{ - Name: ConstructComponentName(typeutil.QueryCoordRole, 1), +func TestQueryCoordTopology_Codec(t *testing.T) { + topology1 := QueryClusterTopology{ + Self: QueryCoordInfos{ + BaseComponentInfos: BaseComponentInfos{ + Name: ConstructComponentName(typeutil.QueryCoordRole, 1), + }, }, - ConnectedNodes: []ComponentInfos{ + ConnectedNodes: []QueryNodeInfos{ { - Name: ConstructComponentName(typeutil.QueryNodeRole, 1), + BaseComponentInfos: BaseComponentInfos{ + Name: ConstructComponentName(typeutil.QueryNodeRole, 1), + }, }, { - Name: ConstructComponentName(typeutil.QueryNodeRole, 2), + BaseComponentInfos: BaseComponentInfos{ + Name: ConstructComponentName(typeutil.QueryNodeRole, 2), + }, }, }, } - s, err := topology1.Marshal() + s, err := MarshalTopology(topology1) assert.Equal(t, nil, err) - var topology2 CoordTopology - err = topology2.Unmarshal(s) + log.Info("TestQueryCoordTopology_Codec", + zap.String("marshaled_result", s)) + var topology2 QueryClusterTopology + err = UnmarshalTopology(s, &topology2) assert.Equal(t, nil, err) - assert.Equal(t, topology1.Self.Name, topology2.Self.Name) + assert.Equal(t, topology1.Self, topology2.Self) assert.Equal(t, len(topology1.ConnectedNodes), len(topology2.ConnectedNodes)) for i := range topology1.ConnectedNodes { assert.Equal(t, topology1.ConnectedNodes[i], topology2.ConnectedNodes[i]) @@ -86,10 +82,12 @@ func TestConnTopology_Codec(t *testing.T) { ConstructComponentName(typeutil.IndexCoordRole, 1), }, } - s, err := topology1.Marshal() + s, err := MarshalTopology(topology1) assert.Equal(t, nil, err) + log.Info("TestConnTopology_Codec", + zap.String("marshaled_result", s)) var topology2 ConnTopology - err = topology2.Unmarshal(s) + err = UnmarshalTopology(s, &topology2) assert.Equal(t, nil, err) assert.Equal(t, topology1.Name, topology2.Name) assert.Equal(t, len(topology1.ConnectedComponents), len(topology2.ConnectedComponents)) @@ -99,10 +97,37 @@ func TestConnTopology_Codec(t *testing.T) { } func TestSystemTopology_Codec(t *testing.T) { - topology1 := SystemTopology{} - s, err := topology1.Marshal() - assert.Equal(t, nil, err) - var topology2 SystemTopology - err = topology2.Unmarshal(s) + topology1 := SystemTopology{ + NodesInfo: []SystemTopologyNode{ + { + Identifier: 1, + Infos: &QueryCoordInfos{ + BaseComponentInfos: BaseComponentInfos{ + Name: ConstructComponentName(typeutil.QueryCoordRole, 1), + }, + }, + Connected: []ConnectionEdge{ + { + ConnectedIdentifier: 2, + Type: CoordConnectToNode, + TargetType: typeutil.QueryNodeRole, + }, + }, + }, + { + Identifier: 2, + Infos: &QueryNodeInfos{ + BaseComponentInfos: BaseComponentInfos{ + Name: ConstructComponentName(typeutil.QueryNodeRole, 2), + }, + }, + Connected: []ConnectionEdge{}, + }, + }, + } + s, err := MarshalTopology(topology1) assert.Equal(t, nil, err) + log.Info("TestSystemTopology_Codec", + zap.String("marshaled_result", s)) + // no need to test unmarshal }