From 0bbb134e398bd9b347617954458af7d5c6f51652 Mon Sep 17 00:00:00 2001 From: XuanYang-cn Date: Tue, 16 Dec 2025 17:19:16 +0800 Subject: [PATCH] feat: Enable to backup and reload ez (#46332) see also: #40013 Signed-off-by: yangxuan --- internal/coordinator/mix_coord.go | 4 + internal/datacoord/compaction_util.go | 31 ++ internal/datacoord/import_util.go | 4 +- internal/datacoord/mock_test.go | 4 + internal/datanode/index_services.go | 19 +- internal/datanode/services.go | 13 +- .../distributed/mixcoord/client/client.go | 11 + internal/distributed/mixcoord/service.go | 4 + internal/distributed/streaming/streaming.go | 2 +- .../flushcommon/syncmgr/pack_writer_v2.go | 2 +- internal/http/router.go | 2 + internal/mocks/mock_mixcoord.go | 59 +++ internal/mocks/mock_mixcoord_client.go | 74 ++++ internal/mocks/mock_rootcoord.go | 59 +++ internal/mocks/mock_rootcoord_client.go | 74 ++++ internal/proxy/management.go | 33 ++ internal/proxy/rootcoord_mock_test.go | 6 + internal/proxy/task_delete_streaming.go | 2 +- internal/proxy/task_insert_streaming.go | 2 +- internal/proxy/task_upsert_streaming.go | 2 +- internal/querynodev2/segments/collection.go | 4 +- internal/rootcoord/create_collection_task.go | 4 +- .../ddl_callbacks_alter_collection_name.go | 2 +- internal/rootcoord/meta_table.go | 2 +- internal/rootcoord/root_coord.go | 42 +++ internal/storage/data_codec.go | 2 +- internal/storage/rw.go | 55 ++- internal/util/hookutil/cipher.go | 230 +++++++----- internal/util/hookutil/cipher_test.go | 213 ++++++++--- internal/util/hookutil/ez.go | 178 +++++++++ internal/util/importutilv2/binlog/reader.go | 28 +- .../util/importutilv2/binlog/reader_test.go | 10 +- internal/util/importutilv2/option.go | 11 + internal/util/importutilv2/reader.go | 3 +- internal/util/initcore/init_core.go | 2 +- internal/util/mock/grpc_rootcoord_client.go | 4 + pkg/proto/internal.proto | 12 +- pkg/proto/internalpb/internal.pb.go | 342 +++++++++++++----- pkg/proto/root_coord.proto | 1 + pkg/proto/rootcoordpb/root_coord.pb.go | 194 +++++----- pkg/proto/rootcoordpb/root_coord_grpc.pb.go | 37 ++ 41 files changed, 1389 insertions(+), 394 deletions(-) create mode 100644 internal/util/hookutil/ez.go diff --git a/internal/coordinator/mix_coord.go b/internal/coordinator/mix_coord.go index b03a8907e8..d4d6e9fd3d 100644 --- a/internal/coordinator/mix_coord.go +++ b/internal/coordinator/mix_coord.go @@ -536,6 +536,10 @@ func (s *mixCoordImpl) DescribeCollectionInternal(ctx context.Context, in *milvu return s.rootcoordServer.DescribeCollectionInternal(ctx, in) } +func (s *mixCoordImpl) BackupEzk(ctx context.Context, in *internalpb.BackupEzkRequest) (*internalpb.BackupEzkResponse, error) { + return s.rootcoordServer.BackupEzk(ctx, in) +} + // DropAlias drop collection alias func (c *mixCoordImpl) DropAlias(ctx context.Context, in *milvuspb.DropAliasRequest) (*commonpb.Status, error) { return c.rootcoordServer.DropAlias(ctx, in) diff --git a/internal/datacoord/compaction_util.go b/internal/datacoord/compaction_util.go index 823a3f8378..ec1f4e808b 100644 --- a/internal/datacoord/compaction_util.go +++ b/internal/datacoord/compaction_util.go @@ -22,6 +22,7 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus/internal/datacoord/allocator" "github.com/milvus-io/milvus/internal/util/hookutil" + "github.com/milvus-io/milvus/internal/util/importutilv2" "github.com/milvus-io/milvus/pkg/v2/proto/datapb" "github.com/milvus-io/milvus/pkg/v2/proto/workerpb" "github.com/milvus-io/milvus/pkg/v2/util/paramtable" @@ -50,6 +51,36 @@ func PreAllocateBinlogIDs(allocator allocator.Allocator, segmentInfos []*Segment return &datapb.IDRange{Begin: begin, End: end}, err } +func WrapPluginContextWithImport(collectionID int64, dbProperties []*commonpb.KeyValuePair, options importutilv2.Options, msg proto.Message) { + pluginContext := make([]*commonpb.KeyValuePair, 0) + + importEzk, _ := importutilv2.GetEZK(options) + readPluginContext := hookutil.GetReadStoragePluginContext(importEzk) + if readPluginContext != nil { + pluginContext = append(pluginContext, readPluginContext...) + } + + writePluginContext := hookutil.GetStoragePluginContext(dbProperties, collectionID) + if writePluginContext != nil { + pluginContext = append(pluginContext, writePluginContext...) + } + + if len(pluginContext) == 0 { + return + } + + switch msg.(type) { + case *datapb.ImportRequest: + job := msg.(*datapb.ImportRequest) + job.PluginContext = append(job.PluginContext, pluginContext...) + case *datapb.PreImportRequest: + job := msg.(*datapb.PreImportRequest) + job.PluginContext = append(job.PluginContext, pluginContext...) + default: + return + } +} + func WrapPluginContext(collectionID int64, properties []*commonpb.KeyValuePair, msg proto.Message) { pluginContext := hookutil.GetStoragePluginContext(properties, collectionID) if pluginContext == nil { diff --git a/internal/datacoord/import_util.go b/internal/datacoord/import_util.go index 80c288c7c2..71ff75a291 100644 --- a/internal/datacoord/import_util.go +++ b/internal/datacoord/import_util.go @@ -301,7 +301,7 @@ func AssemblePreImportRequest(task ImportTask, job ImportJob) *datapb.PreImportR TaskSlot: task.GetTaskSlot(), StorageConfig: createStorageConfig(), } - WrapPluginContext(task.GetCollectionID(), job.GetSchema().GetProperties(), req) + WrapPluginContextWithImport(task.GetCollectionID(), job.GetSchema().GetProperties(), job.GetOptions(), req) return req } @@ -380,7 +380,7 @@ func AssembleImportRequest(task ImportTask, job ImportJob, meta *meta, alloc all TaskSlot: task.GetTaskSlot(), StorageVersion: storageVersion, } - WrapPluginContext(task.GetCollectionID(), job.GetSchema().GetProperties(), req) + WrapPluginContextWithImport(task.GetCollectionID(), job.GetSchema().GetProperties(), job.GetOptions(), req) return req, nil } diff --git a/internal/datacoord/mock_test.go b/internal/datacoord/mock_test.go index 47b6f03199..304976c927 100644 --- a/internal/datacoord/mock_test.go +++ b/internal/datacoord/mock_test.go @@ -505,6 +505,10 @@ func (m *mockMixCoord) RestoreRBAC(ctx context.Context, req *milvuspb.RestoreRBA panic("not implemented") // TODO: Implement } +func (m *mockMixCoord) BackupEzk(ctx context.Context, req *internalpb.BackupEzkRequest) (*internalpb.BackupEzkResponse, error) { + panic("not implemented") // TODO: Implement +} + type mockCompactionTrigger struct { methods map[string]interface{} } diff --git a/internal/datanode/index_services.go b/internal/datanode/index_services.go index f1959e3358..1a36e7ead9 100644 --- a/internal/datanode/index_services.go +++ b/internal/datanode/index_services.go @@ -33,7 +33,6 @@ import ( "github.com/milvus-io/milvus/internal/util/hookutil" "github.com/milvus-io/milvus/pkg/v2/log" "github.com/milvus-io/milvus/pkg/v2/metrics" - "github.com/milvus-io/milvus/pkg/v2/proto/indexcgopb" "github.com/milvus-io/milvus/pkg/v2/proto/indexpb" "github.com/milvus-io/milvus/pkg/v2/proto/workerpb" "github.com/milvus-io/milvus/pkg/v2/util/merr" @@ -100,7 +99,7 @@ func (node *DataNode) CreateJob(ctx context.Context, req *workerpb.CreateJobRequ metrics.DataNodeBuildIndexTaskCounter.WithLabelValues(fmt.Sprint(paramtable.GetNodeID()), metrics.FailLabel).Inc() return merr.Status(err), nil } - pluginContext, err := ParseCPluginContext(req.GetPluginContext(), req.GetCollectionID()) + pluginContext, err := hookutil.GetCPluginContext(req.GetPluginContext(), req.GetCollectionID()) if err != nil { return merr.Status(err), nil } @@ -309,7 +308,7 @@ func (node *DataNode) createIndexTask(ctx context.Context, req *workerpb.CreateJ return merr.Status(err), nil } - pluginContext, err := ParseCPluginContext(req.GetPluginContext(), req.GetCollectionID()) + pluginContext, err := hookutil.GetCPluginContext(req.GetPluginContext(), req.GetCollectionID()) if err != nil { return merr.Status(err), nil } @@ -611,17 +610,3 @@ func (node *DataNode) DropJobsV2(ctx context.Context, req *workerpb.DropJobsV2Re return merr.Status(errors.New("DataNode receive dropping unknown type jobs")), nil } } - -func ParseCPluginContext(context []*commonpb.KeyValuePair, collectionID int64) (*indexcgopb.StoragePluginContext, error) { - pluginContext, err := hookutil.CreateLocalEZByPluginContext(context) - if err != nil { - return nil, err - } - - if pluginContext != nil { - pluginContext.CollectionId = collectionID - return pluginContext, nil - } - - return nil, nil -} diff --git a/internal/datanode/services.go b/internal/datanode/services.go index 50464e170b..2bdacc32c0 100644 --- a/internal/datanode/services.go +++ b/internal/datanode/services.go @@ -569,7 +569,7 @@ func (node *DataNode) CreateTask(ctx context.Context, request *workerpb.CreateTa if err := proto.Unmarshal(request.GetPayload(), req); err != nil { return merr.Status(err), nil } - if _, err := hookutil.CreateLocalEZByPluginContext(req.GetPluginContext()); err != nil { + if err := hookutil.RegisterEZsFromPluginContext(req.GetPluginContext()); err != nil { return merr.Status(err), nil } return node.PreImport(ctx, req) @@ -578,7 +578,7 @@ func (node *DataNode) CreateTask(ctx context.Context, request *workerpb.CreateTa if err := proto.Unmarshal(request.GetPayload(), req); err != nil { return merr.Status(err), nil } - if _, err := hookutil.CreateLocalEZByPluginContext(req.GetPluginContext()); err != nil { + if err := hookutil.RegisterEZsFromPluginContext(req.GetPluginContext()); err != nil { return merr.Status(err), nil } return node.ImportV2(ctx, req) @@ -587,7 +587,7 @@ func (node *DataNode) CreateTask(ctx context.Context, request *workerpb.CreateTa if err := proto.Unmarshal(request.GetPayload(), req); err != nil { return merr.Status(err), nil } - if _, err := hookutil.CreateLocalEZByPluginContext(req.GetPluginContext()); err != nil { + if err := hookutil.RegisterEZsFromPluginContext(req.GetPluginContext()); err != nil { return merr.Status(err), nil } return node.CompactionV2(ctx, req) @@ -596,13 +596,16 @@ func (node *DataNode) CreateTask(ctx context.Context, request *workerpb.CreateTa if err := proto.Unmarshal(request.GetPayload(), req); err != nil { return merr.Status(err), nil } + if err := hookutil.RegisterEZsFromPluginContext(req.GetPluginContext()); err != nil { + return merr.Status(err), nil + } return node.createIndexTask(ctx, req) case taskcommon.Stats: req := &workerpb.CreateStatsRequest{} if err := proto.Unmarshal(request.GetPayload(), req); err != nil { return merr.Status(err), nil } - if _, err := hookutil.CreateLocalEZByPluginContext(req.GetPluginContext()); err != nil { + if err := hookutil.RegisterEZsFromPluginContext(req.GetPluginContext()); err != nil { return merr.Status(err), nil } return node.createStatsTask(ctx, req) @@ -611,7 +614,7 @@ func (node *DataNode) CreateTask(ctx context.Context, request *workerpb.CreateTa if err := proto.Unmarshal(request.GetPayload(), req); err != nil { return merr.Status(err), nil } - if _, err := hookutil.CreateLocalEZByPluginContext(req.GetPluginContext()); err != nil { + if err := hookutil.RegisterEZsFromPluginContext(req.GetPluginContext()); err != nil { return merr.Status(err), nil } return node.createAnalyzeTask(ctx, req) diff --git a/internal/distributed/mixcoord/client/client.go b/internal/distributed/mixcoord/client/client.go index 5814621a24..1631ce56d4 100644 --- a/internal/distributed/mixcoord/client/client.go +++ b/internal/distributed/mixcoord/client/client.go @@ -1972,3 +1972,14 @@ func (c *Client) TruncateCollection(ctx context.Context, in *milvuspb.TruncateCo return client.TruncateCollection(ctx, in) }) } + +func (c *Client) BackupEzk(ctx context.Context, req *internalpb.BackupEzkRequest, opts ...grpc.CallOption) (*internalpb.BackupEzkResponse, error) { + req = typeutil.Clone(req) + commonpbutil.UpdateMsgBase( + req.GetBase(), + commonpbutil.FillMsgBaseFromClient(paramtable.GetNodeID(), commonpbutil.WithTargetID(c.grpcClient.GetNodeID())), + ) + return wrapGrpcCall(ctx, c, func(client MixCoordClient) (*internalpb.BackupEzkResponse, error) { + return client.BackupEzk(ctx, req) + }) +} diff --git a/internal/distributed/mixcoord/service.go b/internal/distributed/mixcoord/service.go index 73dd8a080a..608cd39552 100644 --- a/internal/distributed/mixcoord/service.go +++ b/internal/distributed/mixcoord/service.go @@ -339,6 +339,10 @@ func (s *Server) AlterDatabase(ctx context.Context, request *rootcoordpb.AlterDa return s.mixCoord.AlterDatabase(ctx, request) } +func (s *Server) BackupEzk(ctx context.Context, request *internalpb.BackupEzkRequest) (*internalpb.BackupEzkResponse, error) { + return s.mixCoord.BackupEzk(ctx, request) +} + func (s *Server) CheckHealth(ctx context.Context, request *milvuspb.CheckHealthRequest) (*milvuspb.CheckHealthResponse, error) { return s.mixCoord.CheckHealth(ctx, request) } diff --git a/internal/distributed/streaming/streaming.go b/internal/distributed/streaming/streaming.go index 10f9cb9123..dc610134f4 100644 --- a/internal/distributed/streaming/streaming.go +++ b/internal/distributed/streaming/streaming.go @@ -23,7 +23,7 @@ func Init() { // init and select wal name util.InitAndSelectWALName() // register cipher for cipher message - if hookutil.IsClusterEncyptionEnabled() { + if hookutil.IsClusterEncryptionEnabled() { message.RegisterCipher(hookutil.GetCipher()) } singleton = newWALAccesser(c) diff --git a/internal/flushcommon/syncmgr/pack_writer_v2.go b/internal/flushcommon/syncmgr/pack_writer_v2.go index ef11cde774..d39a01ff02 100644 --- a/internal/flushcommon/syncmgr/pack_writer_v2.go +++ b/internal/flushcommon/syncmgr/pack_writer_v2.go @@ -149,7 +149,7 @@ func (bw *BulkPackWriterV2) writeInserts(ctx context.Context, pack *SyncPack) (m } } var pluginContextPtr *indexcgopb.StoragePluginContext - if hookutil.IsClusterEncyptionEnabled() { + if hookutil.IsClusterEncryptionEnabled() { ez := hookutil.GetEzByCollProperties(bw.schema.GetProperties(), pack.collectionID) if ez != nil { unsafe := hookutil.GetCipher().GetUnsafeKey(ez.EzID, ez.CollectionID) diff --git a/internal/http/router.go b/internal/http/router.go index 385c144801..6d4951ad7d 100644 --- a/internal/http/router.go +++ b/internal/http/router.go @@ -51,6 +51,8 @@ const ( // proxy management restful api root path const ( + RouteBackupEZ = "/management/rootcoord/ez/backup" + RouteGcPause = "/management/datacoord/garbage_collection/pause" RouteGcResume = "/management/datacoord/garbage_collection/resume" diff --git a/internal/mocks/mock_mixcoord.go b/internal/mocks/mock_mixcoord.go index 93995dc8fa..80752da0d4 100644 --- a/internal/mocks/mock_mixcoord.go +++ b/internal/mocks/mock_mixcoord.go @@ -872,6 +872,65 @@ func (_c *MixCoord_AssignSegmentID_Call) RunAndReturn(run func(context.Context, return _c } +// BackupEzk provides a mock function with given fields: _a0, _a1 +func (_m *MixCoord) BackupEzk(_a0 context.Context, _a1 *internalpb.BackupEzkRequest) (*internalpb.BackupEzkResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for BackupEzk") + } + + var r0 *internalpb.BackupEzkResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *internalpb.BackupEzkRequest) (*internalpb.BackupEzkResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *internalpb.BackupEzkRequest) *internalpb.BackupEzkResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*internalpb.BackupEzkResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *internalpb.BackupEzkRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MixCoord_BackupEzk_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'BackupEzk' +type MixCoord_BackupEzk_Call struct { + *mock.Call +} + +// BackupEzk is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *internalpb.BackupEzkRequest +func (_e *MixCoord_Expecter) BackupEzk(_a0 interface{}, _a1 interface{}) *MixCoord_BackupEzk_Call { + return &MixCoord_BackupEzk_Call{Call: _e.mock.On("BackupEzk", _a0, _a1)} +} + +func (_c *MixCoord_BackupEzk_Call) Run(run func(_a0 context.Context, _a1 *internalpb.BackupEzkRequest)) *MixCoord_BackupEzk_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*internalpb.BackupEzkRequest)) + }) + return _c +} + +func (_c *MixCoord_BackupEzk_Call) Return(_a0 *internalpb.BackupEzkResponse, _a1 error) *MixCoord_BackupEzk_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MixCoord_BackupEzk_Call) RunAndReturn(run func(context.Context, *internalpb.BackupEzkRequest) (*internalpb.BackupEzkResponse, error)) *MixCoord_BackupEzk_Call { + _c.Call.Return(run) + return _c +} + // BackupRBAC provides a mock function with given fields: _a0, _a1 func (_m *MixCoord) BackupRBAC(_a0 context.Context, _a1 *milvuspb.BackupRBACMetaRequest) (*milvuspb.BackupRBACMetaResponse, error) { ret := _m.Called(_a0, _a1) diff --git a/internal/mocks/mock_mixcoord_client.go b/internal/mocks/mock_mixcoord_client.go index 5109b6b0e4..2e32e6d3a7 100644 --- a/internal/mocks/mock_mixcoord_client.go +++ b/internal/mocks/mock_mixcoord_client.go @@ -1077,6 +1077,80 @@ func (_c *MockMixCoordClient_AssignSegmentID_Call) RunAndReturn(run func(context return _c } +// BackupEzk provides a mock function with given fields: ctx, in, opts +func (_m *MockMixCoordClient) BackupEzk(ctx context.Context, in *internalpb.BackupEzkRequest, opts ...grpc.CallOption) (*internalpb.BackupEzkResponse, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, in) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for BackupEzk") + } + + var r0 *internalpb.BackupEzkResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *internalpb.BackupEzkRequest, ...grpc.CallOption) (*internalpb.BackupEzkResponse, error)); ok { + return rf(ctx, in, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, *internalpb.BackupEzkRequest, ...grpc.CallOption) *internalpb.BackupEzkResponse); ok { + r0 = rf(ctx, in, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*internalpb.BackupEzkResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *internalpb.BackupEzkRequest, ...grpc.CallOption) error); ok { + r1 = rf(ctx, in, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockMixCoordClient_BackupEzk_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'BackupEzk' +type MockMixCoordClient_BackupEzk_Call struct { + *mock.Call +} + +// BackupEzk is a helper method to define mock.On call +// - ctx context.Context +// - in *internalpb.BackupEzkRequest +// - opts ...grpc.CallOption +func (_e *MockMixCoordClient_Expecter) BackupEzk(ctx interface{}, in interface{}, opts ...interface{}) *MockMixCoordClient_BackupEzk_Call { + return &MockMixCoordClient_BackupEzk_Call{Call: _e.mock.On("BackupEzk", + append([]interface{}{ctx, in}, opts...)...)} +} + +func (_c *MockMixCoordClient_BackupEzk_Call) Run(run func(ctx context.Context, in *internalpb.BackupEzkRequest, opts ...grpc.CallOption)) *MockMixCoordClient_BackupEzk_Call { + _c.Call.Run(func(args mock.Arguments) { + variadicArgs := make([]grpc.CallOption, len(args)-2) + for i, a := range args[2:] { + if a != nil { + variadicArgs[i] = a.(grpc.CallOption) + } + } + run(args[0].(context.Context), args[1].(*internalpb.BackupEzkRequest), variadicArgs...) + }) + return _c +} + +func (_c *MockMixCoordClient_BackupEzk_Call) Return(_a0 *internalpb.BackupEzkResponse, _a1 error) *MockMixCoordClient_BackupEzk_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockMixCoordClient_BackupEzk_Call) RunAndReturn(run func(context.Context, *internalpb.BackupEzkRequest, ...grpc.CallOption) (*internalpb.BackupEzkResponse, error)) *MockMixCoordClient_BackupEzk_Call { + _c.Call.Return(run) + return _c +} + // BackupRBAC provides a mock function with given fields: ctx, in, opts func (_m *MockMixCoordClient) BackupRBAC(ctx context.Context, in *milvuspb.BackupRBACMetaRequest, opts ...grpc.CallOption) (*milvuspb.BackupRBACMetaResponse, error) { _va := make([]interface{}, len(opts)) diff --git a/internal/mocks/mock_rootcoord.go b/internal/mocks/mock_rootcoord.go index aed13709b1..3a64b50e5a 100644 --- a/internal/mocks/mock_rootcoord.go +++ b/internal/mocks/mock_rootcoord.go @@ -567,6 +567,65 @@ func (_c *MockRootCoord_AlterDatabase_Call) RunAndReturn(run func(context.Contex return _c } +// BackupEzk provides a mock function with given fields: _a0, _a1 +func (_m *MockRootCoord) BackupEzk(_a0 context.Context, _a1 *internalpb.BackupEzkRequest) (*internalpb.BackupEzkResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for BackupEzk") + } + + var r0 *internalpb.BackupEzkResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *internalpb.BackupEzkRequest) (*internalpb.BackupEzkResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *internalpb.BackupEzkRequest) *internalpb.BackupEzkResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*internalpb.BackupEzkResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *internalpb.BackupEzkRequest) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockRootCoord_BackupEzk_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'BackupEzk' +type MockRootCoord_BackupEzk_Call struct { + *mock.Call +} + +// BackupEzk is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *internalpb.BackupEzkRequest +func (_e *MockRootCoord_Expecter) BackupEzk(_a0 interface{}, _a1 interface{}) *MockRootCoord_BackupEzk_Call { + return &MockRootCoord_BackupEzk_Call{Call: _e.mock.On("BackupEzk", _a0, _a1)} +} + +func (_c *MockRootCoord_BackupEzk_Call) Run(run func(_a0 context.Context, _a1 *internalpb.BackupEzkRequest)) *MockRootCoord_BackupEzk_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*internalpb.BackupEzkRequest)) + }) + return _c +} + +func (_c *MockRootCoord_BackupEzk_Call) Return(_a0 *internalpb.BackupEzkResponse, _a1 error) *MockRootCoord_BackupEzk_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockRootCoord_BackupEzk_Call) RunAndReturn(run func(context.Context, *internalpb.BackupEzkRequest) (*internalpb.BackupEzkResponse, error)) *MockRootCoord_BackupEzk_Call { + _c.Call.Return(run) + return _c +} + // BackupRBAC provides a mock function with given fields: _a0, _a1 func (_m *MockRootCoord) BackupRBAC(_a0 context.Context, _a1 *milvuspb.BackupRBACMetaRequest) (*milvuspb.BackupRBACMetaResponse, error) { ret := _m.Called(_a0, _a1) diff --git a/internal/mocks/mock_rootcoord_client.go b/internal/mocks/mock_rootcoord_client.go index 441560ed36..908198d29b 100644 --- a/internal/mocks/mock_rootcoord_client.go +++ b/internal/mocks/mock_rootcoord_client.go @@ -699,6 +699,80 @@ func (_c *MockRootCoordClient_AlterDatabase_Call) RunAndReturn(run func(context. return _c } +// BackupEzk provides a mock function with given fields: ctx, in, opts +func (_m *MockRootCoordClient) BackupEzk(ctx context.Context, in *internalpb.BackupEzkRequest, opts ...grpc.CallOption) (*internalpb.BackupEzkResponse, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, in) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for BackupEzk") + } + + var r0 *internalpb.BackupEzkResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *internalpb.BackupEzkRequest, ...grpc.CallOption) (*internalpb.BackupEzkResponse, error)); ok { + return rf(ctx, in, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, *internalpb.BackupEzkRequest, ...grpc.CallOption) *internalpb.BackupEzkResponse); ok { + r0 = rf(ctx, in, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*internalpb.BackupEzkResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *internalpb.BackupEzkRequest, ...grpc.CallOption) error); ok { + r1 = rf(ctx, in, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockRootCoordClient_BackupEzk_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'BackupEzk' +type MockRootCoordClient_BackupEzk_Call struct { + *mock.Call +} + +// BackupEzk is a helper method to define mock.On call +// - ctx context.Context +// - in *internalpb.BackupEzkRequest +// - opts ...grpc.CallOption +func (_e *MockRootCoordClient_Expecter) BackupEzk(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_BackupEzk_Call { + return &MockRootCoordClient_BackupEzk_Call{Call: _e.mock.On("BackupEzk", + append([]interface{}{ctx, in}, opts...)...)} +} + +func (_c *MockRootCoordClient_BackupEzk_Call) Run(run func(ctx context.Context, in *internalpb.BackupEzkRequest, opts ...grpc.CallOption)) *MockRootCoordClient_BackupEzk_Call { + _c.Call.Run(func(args mock.Arguments) { + variadicArgs := make([]grpc.CallOption, len(args)-2) + for i, a := range args[2:] { + if a != nil { + variadicArgs[i] = a.(grpc.CallOption) + } + } + run(args[0].(context.Context), args[1].(*internalpb.BackupEzkRequest), variadicArgs...) + }) + return _c +} + +func (_c *MockRootCoordClient_BackupEzk_Call) Return(_a0 *internalpb.BackupEzkResponse, _a1 error) *MockRootCoordClient_BackupEzk_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockRootCoordClient_BackupEzk_Call) RunAndReturn(run func(context.Context, *internalpb.BackupEzkRequest, ...grpc.CallOption) (*internalpb.BackupEzkResponse, error)) *MockRootCoordClient_BackupEzk_Call { + _c.Call.Return(run) + return _c +} + // BackupRBAC provides a mock function with given fields: ctx, in, opts func (_m *MockRootCoordClient) BackupRBAC(ctx context.Context, in *milvuspb.BackupRBACMetaRequest, opts ...grpc.CallOption) (*milvuspb.BackupRBACMetaResponse, error) { _va := make([]interface{}, len(opts)) diff --git a/internal/proxy/management.go b/internal/proxy/management.go index 3d6bd46f4b..1a3b8fa72e 100644 --- a/internal/proxy/management.go +++ b/internal/proxy/management.go @@ -30,6 +30,7 @@ import ( management "github.com/milvus-io/milvus/internal/http" "github.com/milvus-io/milvus/internal/json" "github.com/milvus-io/milvus/pkg/v2/proto/datapb" + "github.com/milvus-io/milvus/pkg/v2/proto/internalpb" "github.com/milvus-io/milvus/pkg/v2/proto/querypb" "github.com/milvus-io/milvus/pkg/v2/util/commonpbutil" "github.com/milvus-io/milvus/pkg/v2/util/merr" @@ -88,6 +89,10 @@ func RegisterMgrRoute(proxy *Proxy) { Path: management.RouteQueryCoordBalanceStatus, HandlerFunc: proxy.CheckQueryCoordBalanceStatus, }) + management.Register(&management.Handler{ + Path: management.RouteBackupEZ, + HandlerFunc: proxy.BackupEZ, + }) }) } @@ -585,3 +590,31 @@ func (node *Proxy) CheckQueryNodeDistribution(w http.ResponseWriter, req *http.R w.WriteHeader(http.StatusOK) w.Write([]byte(`{"msg": "OK"}`)) } + +func (node *Proxy) BackupEZ(w http.ResponseWriter, req *http.Request) { + dbName := req.URL.Query().Get("db_name") + if dbName == "" { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte(`{"msg": "db_name parameter is required"}`)) + return + } + + resp, err := node.mixCoord.BackupEzk(req.Context(), &internalpb.BackupEzkRequest{ + Base: commonpbutil.NewMsgBase(), + DbName: dbName, + }) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(fmt.Sprintf(`{"msg": "failed to backup EZK, %s"}`, err.Error()))) + return + } + + if !merr.Ok(resp.GetStatus()) { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte(fmt.Sprintf(`{"msg": "failed to backup EZK, %s"}`, resp.GetStatus().GetReason()))) + return + } + + w.WriteHeader(http.StatusOK) + w.Write([]byte(fmt.Sprintf(`{"msg": "OK", "ezk": "%s"}`, resp.Ezk))) +} diff --git a/internal/proxy/rootcoord_mock_test.go b/internal/proxy/rootcoord_mock_test.go index 0cc88b7740..1d297444f4 100644 --- a/internal/proxy/rootcoord_mock_test.go +++ b/internal/proxy/rootcoord_mock_test.go @@ -1677,6 +1677,12 @@ func (coord *MixCoordMock) TruncateCollection(ctx context.Context, req *milvuspb }, nil } +func (coord *MixCoordMock) BackupEzk(ctx context.Context, req *internalpb.BackupEzkRequest, opts ...grpc.CallOption) (*internalpb.BackupEzkResponse, error) { + return &internalpb.BackupEzkResponse{ + Status: merr.Success(), + }, nil +} + type DescribeCollectionFunc func(ctx context.Context, request *milvuspb.DescribeCollectionRequest, opts ...grpc.CallOption) (*milvuspb.DescribeCollectionResponse, error) type ShowPartitionsFunc func(ctx context.Context, request *milvuspb.ShowPartitionsRequest, opts ...grpc.CallOption) (*milvuspb.ShowPartitionsResponse, error) diff --git a/internal/proxy/task_delete_streaming.go b/internal/proxy/task_delete_streaming.go index 88a29ae811..7ceb5bd060 100644 --- a/internal/proxy/task_delete_streaming.go +++ b/internal/proxy/task_delete_streaming.go @@ -40,7 +40,7 @@ func (dt *deleteTask) Execute(ctx context.Context) (err error) { } var ez *message.CipherConfig - if hookutil.IsClusterEncyptionEnabled() { + if hookutil.IsClusterEncryptionEnabled() { schema, err := globalMetaCache.GetCollectionSchema(ctx, dt.req.GetDbName(), dt.req.GetCollectionName()) if err != nil { log.Ctx(ctx).Warn("get collection schema from global meta cache failed", zap.String("collectionName", dt.req.GetCollectionName()), zap.Error(err)) diff --git a/internal/proxy/task_insert_streaming.go b/internal/proxy/task_insert_streaming.go index 2cd08b8909..187d78c2e1 100644 --- a/internal/proxy/task_insert_streaming.go +++ b/internal/proxy/task_insert_streaming.go @@ -52,7 +52,7 @@ func (it *insertTask) Execute(ctx context.Context) error { zap.Duration("get cache duration", getCacheDur)) var ez *message.CipherConfig - if hookutil.IsClusterEncyptionEnabled() { + if hookutil.IsClusterEncryptionEnabled() { ez = hookutil.GetEzByCollProperties(it.schema.GetProperties(), it.collectionID).AsMessageConfig() } diff --git a/internal/proxy/task_upsert_streaming.go b/internal/proxy/task_upsert_streaming.go index 0e146f8626..f3fc6d94cd 100644 --- a/internal/proxy/task_upsert_streaming.go +++ b/internal/proxy/task_upsert_streaming.go @@ -22,7 +22,7 @@ func (ut *upsertTask) Execute(ctx context.Context) error { log := log.Ctx(ctx).With(zap.String("collectionName", ut.req.CollectionName)) var ez *message.CipherConfig - if hookutil.IsClusterEncyptionEnabled() { + if hookutil.IsClusterEncryptionEnabled() { ez = hookutil.GetEzByCollProperties(ut.schema.GetProperties(), ut.collectionID).AsMessageConfig() } diff --git a/internal/querynodev2/segments/collection.go b/internal/querynodev2/segments/collection.go index d84c85800d..306b514cb2 100644 --- a/internal/querynodev2/segments/collection.go +++ b/internal/querynodev2/segments/collection.go @@ -378,7 +378,7 @@ func DeleteCollection(collection *Collection) { collection.mu.Lock() defer collection.mu.Unlock() - if hookutil.IsClusterEncyptionEnabled() { + if hookutil.IsClusterEncryptionEnabled() { ez := hookutil.GetEzByCollProperties(collection.Schema().GetProperties(), collection.ID()) if ez != nil { if err := segcore.UnRefPluginContext(ez); err != nil { @@ -395,7 +395,7 @@ func DeleteCollection(collection *Collection) { } func putOrUpdateStorageContext(properties []*commonpb.KeyValuePair, collectionID int64) { - if hookutil.IsClusterEncyptionEnabled() { + if hookutil.IsClusterEncryptionEnabled() { ez := hookutil.GetEzByCollProperties(properties, collectionID) if ez != nil { key := hookutil.GetCipher().GetUnsafeKey(ez.EzID, ez.CollectionID) diff --git a/internal/rootcoord/create_collection_task.go b/internal/rootcoord/create_collection_task.go index 56a8600979..46d0beb986 100644 --- a/internal/rootcoord/create_collection_task.go +++ b/internal/rootcoord/create_collection_task.go @@ -486,9 +486,7 @@ func (t *createCollectionTask) Prepare(ctx context.Context) error { t.Req.Properties = append(properties, timezoneKV) } - if ezProps := hookutil.GetEzPropByDBProperties(db.Properties); ezProps != nil { - t.Req.Properties = append(t.Req.Properties, ezProps) - } + t.Req.Properties = hookutil.TidyCollPropsByDBProps(t.Req.Properties, db.Properties) t.header.DbId = db.ID t.body.DbID = t.header.DbId diff --git a/internal/rootcoord/ddl_callbacks_alter_collection_name.go b/internal/rootcoord/ddl_callbacks_alter_collection_name.go index 3c23cf0475..f769611a50 100644 --- a/internal/rootcoord/ddl_callbacks_alter_collection_name.go +++ b/internal/rootcoord/ddl_callbacks_alter_collection_name.go @@ -119,7 +119,7 @@ func (c *Core) validateEncryption(ctx context.Context, oldDBName string, newDBNa } // Check if either database has encryption enabled - if hookutil.IsDBEncryptionEnabled(originalDB.Properties) || hookutil.IsDBEncryptionEnabled(targetDB.Properties) { + if hookutil.IsDBEncrypted(originalDB.Properties) || hookutil.IsDBEncrypted(targetDB.Properties) { return fmt.Errorf("deny to change collection databases due to at least one database enabled encryption, original DB: %s, target DB: %s", oldDBName, newDBName) } diff --git a/internal/rootcoord/meta_table.go b/internal/rootcoord/meta_table.go index d10d863aac..31f001e502 100644 --- a/internal/rootcoord/meta_table.go +++ b/internal/rootcoord/meta_table.go @@ -331,7 +331,7 @@ func (mt *MetaTable) createDefaultDb() error { } defaultRootKey := paramtable.GetCipherParams().DefaultRootKey.GetValue() - if hookutil.IsClusterEncyptionEnabled() && len(defaultRootKey) > 0 { + if hookutil.IsClusterEncryptionEnabled() && len(defaultRootKey) > 0 { // Set unique ID as ezID because the default dbID for each cluster // is the same ezID, err := mt.tsoAllocator.GenerateTSO(1) diff --git a/internal/rootcoord/root_coord.go b/internal/rootcoord/root_coord.go index d4302ff14c..9543ff01ba 100644 --- a/internal/rootcoord/root_coord.go +++ b/internal/rootcoord/root_coord.go @@ -48,6 +48,7 @@ import ( tso2 "github.com/milvus-io/milvus/internal/tso" "github.com/milvus-io/milvus/internal/types" "github.com/milvus-io/milvus/internal/util/dependency" + "github.com/milvus-io/milvus/internal/util/hookutil" "github.com/milvus-io/milvus/internal/util/proxyutil" "github.com/milvus-io/milvus/internal/util/sessionutil" "github.com/milvus-io/milvus/internal/util/streamingutil" @@ -3128,3 +3129,44 @@ func isVisibleCollectionForCurUser(collectionName string, visibleCollections typ func (c *Core) GetQuotaMetrics(ctx context.Context, req *internalpb.GetQuotaMetricsRequest) (*internalpb.GetQuotaMetricsResponse, error) { return c.quotaCenter.getQuotaMetrics(), nil } + +func (c *Core) BackupEzk(ctx context.Context, req *internalpb.BackupEzkRequest) (*internalpb.BackupEzkResponse, error) { + method := "BackupEzk" + metrics.RootCoordDDLReqCounter.WithLabelValues(method, metrics.TotalLabel).Inc() + tr := timerecord.NewTimeRecorder(method) + ctxLog := log.Ctx(ctx).With(zap.String("role", typeutil.RootCoordRole), zap.String("dbName", req.GetDbName())) + ctxLog.Debug(method) + + if err := merr.CheckHealthy(c.GetStateCode()); err != nil { + return &internalpb.BackupEzkResponse{ + Status: merr.Status(err), + }, nil + } + + db, err := c.meta.GetDatabaseByName(ctx, req.GetDbName(), 0) + if err != nil { + ctxLog.Warn("database not found", zap.Error(err)) + metrics.RootCoordDDLReqCounter.WithLabelValues(method, metrics.FailLabel).Inc() + return &internalpb.BackupEzkResponse{ + Status: merr.Status(err), + }, nil + } + + ezkJSON, err := hookutil.BackupEZKFromDBProperties(db.Properties) + if err != nil { + ctxLog.Warn("failed to backup EZK", zap.Error(err)) + metrics.RootCoordDDLReqCounter.WithLabelValues(method, metrics.FailLabel).Inc() + return &internalpb.BackupEzkResponse{ + Status: merr.Status(err), + Ezk: "", + }, nil + } + + ctxLog.Debug(method + " success") + metrics.RootCoordDDLReqCounter.WithLabelValues(method, metrics.SuccessLabel).Inc() + metrics.RootCoordDDLReqLatency.WithLabelValues(method).Observe(float64(tr.ElapseSpan().Milliseconds())) + return &internalpb.BackupEzkResponse{ + Status: merr.Success(), + Ezk: ezkJSON, + }, nil +} diff --git a/internal/storage/data_codec.go b/internal/storage/data_codec.go index aa0afd1806..6e94131485 100644 --- a/internal/storage/data_codec.go +++ b/internal/storage/data_codec.go @@ -257,7 +257,7 @@ func (insertCodec *InsertCodec) Serialize(partitionID UniqueID, segmentID Unique } binlogWriterOpts := []BinlogWriterOptions{} - if hookutil.IsClusterEncyptionEnabled() { + if hookutil.IsClusterEncryptionEnabled() { if ez := hookutil.GetEzByCollProperties(insertCodec.Schema.GetSchema().GetProperties(), insertCodec.Schema.ID); ez != nil { encryptor, safeKey, err := hookutil.GetCipher().GetEncryptor(ez.EzID, ez.CollectionID) if err != nil { diff --git a/internal/storage/rw.go b/internal/storage/rw.go index 1ea675d041..2952c55481 100644 --- a/internal/storage/rw.go +++ b/internal/storage/rw.go @@ -69,6 +69,7 @@ type rwOptions struct { storageConfig *indexpb.StorageConfig neededFields typeutil.Set[int64] useLoonFFI bool + pluginContext *indexcgopb.StoragePluginContext } func (o *rwOptions) validate() error { @@ -171,6 +172,12 @@ func WithUseLoonFFI(useLoonFFI bool) RwOption { } } +func WithPluginContext(pluginContext *indexcgopb.StoragePluginContext) RwOption { + return func(options *rwOptions) { + options.pluginContext = pluginContext + } +} + func makeBlobsReader(ctx context.Context, binlogs []*datapb.FieldBinlog, downloader downloaderFn) (ChunkedBlobsReader, error) { if len(binlogs) == 0 { return func() ([]*Blob, error) { @@ -253,16 +260,22 @@ func NewBinlogRecordReader(ctx context.Context, binlogs []*datapb.FieldBinlog, s binlogReaderOpts := []BinlogReaderOption{} var pluginContext *indexcgopb.StoragePluginContext - if hookutil.IsClusterEncyptionEnabled() { - if ez := hookutil.GetEzByCollProperties(schema.GetProperties(), rwOptions.collectionID); ez != nil { - binlogReaderOpts = append(binlogReaderOpts, WithReaderDecryptionContext(ez.EzID, ez.CollectionID)) + if hookutil.IsClusterEncryptionEnabled() { + // Reader pluginContext from import tasks + if rwOptions.pluginContext != nil { + pluginContext = rwOptions.pluginContext + } else { + ez := hookutil.GetEzByCollProperties(schema.GetProperties(), rwOptions.collectionID) + if ez != nil { + binlogReaderOpts = append(binlogReaderOpts, WithReaderDecryptionContext(ez.EzID, ez.CollectionID)) - unsafe := hookutil.GetCipher().GetUnsafeKey(ez.EzID, ez.CollectionID) - if len(unsafe) > 0 { - pluginContext = &indexcgopb.StoragePluginContext{ - EncryptionZoneId: ez.EzID, - CollectionId: ez.CollectionID, - EncryptionKey: base64.StdEncoding.EncodeToString(unsafe), + unsafe := hookutil.GetCipher().GetUnsafeKey(ez.EzID, ez.CollectionID) + if len(unsafe) > 0 { + pluginContext = &indexcgopb.StoragePluginContext{ + EncryptionZoneId: ez.EzID, + CollectionId: ez.CollectionID, + EncryptionKey: base64.StdEncoding.EncodeToString(unsafe), + } } } } @@ -318,14 +331,20 @@ func NewManifestRecordReader(ctx context.Context, manifestPath string, schema *s } var pluginContext *indexcgopb.StoragePluginContext - if hookutil.IsClusterEncyptionEnabled() { - if ez := hookutil.GetEzByCollProperties(schema.GetProperties(), rwOptions.collectionID); ez != nil { - unsafe := hookutil.GetCipher().GetUnsafeKey(ez.EzID, ez.CollectionID) - if len(unsafe) > 0 { - pluginContext = &indexcgopb.StoragePluginContext{ - EncryptionZoneId: ez.EzID, - CollectionId: ez.CollectionID, - EncryptionKey: string(unsafe), + if hookutil.IsClusterEncryptionEnabled() { + // Reader pluginContext from import tasks + if rwOptions.pluginContext != nil { + pluginContext = rwOptions.pluginContext + } else { + ez := hookutil.GetEzByCollProperties(schema.GetProperties(), rwOptions.collectionID) + if ez != nil { + unsafe := hookutil.GetCipher().GetUnsafeKey(ez.EzID, ez.CollectionID) + if len(unsafe) > 0 { + pluginContext = &indexcgopb.StoragePluginContext{ + EncryptionZoneId: ez.EzID, + CollectionId: ez.CollectionID, + EncryptionKey: string(unsafe), + } } } } @@ -357,7 +376,7 @@ func NewBinlogRecordWriter(ctx context.Context, collectionID, partitionID, segme opts := []StreamWriterOption{} var pluginContext *indexcgopb.StoragePluginContext - if hookutil.IsClusterEncyptionEnabled() { + if hookutil.IsClusterEncryptionEnabled() { ez := hookutil.GetEzByCollProperties(schema.GetProperties(), collectionID) if ez != nil { encryptor, edek, err := hookutil.GetCipher().GetEncryptor(ez.EzID, ez.CollectionID) diff --git a/internal/util/hookutil/cipher.go b/internal/util/hookutil/cipher.go index d11174ff67..46fd67e413 100644 --- a/internal/util/hookutil/cipher.go +++ b/internal/util/hookutil/cipher.go @@ -46,6 +46,10 @@ var ( ErrCipherPluginMissing = errors.New("cipher plugin is missing") ) +type BackupInterface interface { + Backup(ezID int64) (string, error) +} + // GetCipher returns singleton hook.Cipher instance. // If Milvus is not built with cipher plugin, it will return nil // If Milvus is built with cipher plugin, it will return hook.Cipher @@ -54,7 +58,7 @@ func GetCipher() hook.Cipher { return Cipher.Load().(cipherContainer).cipher } -func IsClusterEncyptionEnabled() bool { +func IsClusterEncryptionEnabled() bool { return GetCipher() != nil } @@ -63,13 +67,6 @@ const ( EncryptionEnabledKey = "cipher.enabled" EncryptionRootKeyKey = "cipher.key" EncryptionEzIDKey = "cipher.ezID" - - // Used in Plugins - CipherConfigCreateEZ = "cipher.ez.create" - CipherConfigRemoveEZ = "cipher.ez.remove" - CipherConfigMilvusRoleName = "cipher.milvusRoleName" - CipherConfigKeyKmsKeyArn = "cipher.kmsKeyArn" - CipherConfigUnsafeEZK = "cipher.ezk" ) type EZ struct { @@ -118,30 +115,6 @@ func GetEzByCollProperties(collProperties []*commonpb.KeyValuePair, collectionID return nil } -// GetStoragePluginContext returns the local plugin context for RPC from datacoord to datanode -func GetStoragePluginContext(properties []*commonpb.KeyValuePair, collectionID int64) []*commonpb.KeyValuePair { - if GetCipher() == nil { - return nil - } - - if ez := GetEzByCollProperties(properties, collectionID); ez != nil { - key := GetCipher().GetUnsafeKey(ez.EzID, ez.CollectionID) - pluginContext := []*commonpb.KeyValuePair{ - { - Key: CipherConfigCreateEZ, - Value: strconv.FormatInt(ez.EzID, 10), - }, - { - Key: CipherConfigUnsafeEZK, - Value: base64.StdEncoding.EncodeToString(key), - }, - } - return pluginContext - } - - return nil -} - func GetDBCipherProperties(ezID uint64, kmsKey string) []*commonpb.KeyValuePair { return []*commonpb.KeyValuePair{ { @@ -159,78 +132,27 @@ func GetDBCipherProperties(ezID uint64, kmsKey string) []*commonpb.KeyValuePair } } -func RemoveEZByDBProperties(dbProperties []*commonpb.KeyValuePair) error { - if GetCipher() == nil { - return nil - } - - ezIdStr := "" - for _, property := range dbProperties { - if property.Key == EncryptionEzIDKey { - ezIdStr = property.Value - } - } - if len(ezIdStr) == 0 { - return nil - } - - dropConfig := map[string]string{CipherConfigRemoveEZ: ezIdStr} - if err := GetCipher().Init(dropConfig); err != nil { - return err - } - return nil -} - -func CreateLocalEZByPluginContext(context []*commonpb.KeyValuePair) (*indexcgopb.StoragePluginContext, error) { - if GetCipher() == nil { - return nil, nil - } - config := make(map[string]string) - ctx := &indexcgopb.StoragePluginContext{} - for _, value := range context { - if value.GetKey() == CipherConfigCreateEZ { - ezID, err := strconv.ParseInt(value.GetValue(), 10, 64) - if err != nil { - return nil, err - } - config[CipherConfigCreateEZ] = value.GetValue() - ctx.EncryptionZoneId = ezID - } - if value.GetKey() == CipherConfigUnsafeEZK { - config[CipherConfigUnsafeEZK] = value.GetValue() - ctx.EncryptionKey = value.GetValue() - } - } - if len(config) == 2 { - return ctx, GetCipher().Init(config) - } - return nil, nil -} - func CreateEZByDBProperties(dbProperties []*commonpb.KeyValuePair) error { - if GetCipher() == nil { + ezID, hasEzID := ParseEzIDFromProperties(dbProperties) + if !hasEzID { return nil } - config := make(map[string]string) for _, property := range dbProperties { - if property.GetKey() == EncryptionEzIDKey { - config[CipherConfigCreateEZ] = property.Value - } if property.GetKey() == EncryptionRootKeyKey { - config[CipherConfigKeyKmsKeyArn] = property.GetValue() + return CreateEZ(ezID, property.GetValue()) } } - if len(config) == 2 { - return GetCipher().Init(config) - } - return nil } +// When creating a new database with encryption eabled, set the ezID to the dbProperties, +// and try to use the the config rootKey if rootKey not provided +// An encrypted DB's properties will contain three properties: +// cipher.enabled, cipher.ezID, cipher.key func TidyDBCipherProperties(ezID int64, dbProperties []*commonpb.KeyValuePair) ([]*commonpb.KeyValuePair, error) { - dbEncryptionEnabled := IsDBEncryptionEnabled(dbProperties) + dbEncryptionEnabled := IsDBEncrypted(dbProperties) if GetCipher() == nil { if dbEncryptionEnabled { return nil, ErrCipherPluginMissing @@ -267,7 +189,24 @@ func TidyDBCipherProperties(ezID int64, dbProperties []*commonpb.KeyValuePair) ( return dbProperties, nil } -func GetEzPropByDBProperties(dbProperties []*commonpb.KeyValuePair) *commonpb.KeyValuePair { +func TidyCollPropsByDBProps(collProps, dbProps []*commonpb.KeyValuePair) []*commonpb.KeyValuePair { + newCollProps := []*commonpb.KeyValuePair{} + for _, property := range collProps { + // Ignore already have ez property, likely from backup collection's schema + if property.Key == EncryptionEzIDKey { + continue + } + newCollProps = append(newCollProps, property) + } + + // Set the new database's encryption properties + if ezProps := getCollEzPropsByDBProps(dbProps); ezProps != nil { + newCollProps = append(newCollProps, ezProps) + } + return newCollProps +} + +func getCollEzPropsByDBProps(dbProperties []*commonpb.KeyValuePair) *commonpb.KeyValuePair { for _, property := range dbProperties { if property.Key == EncryptionEzIDKey { return &commonpb.KeyValuePair{ @@ -279,7 +218,7 @@ func GetEzPropByDBProperties(dbProperties []*commonpb.KeyValuePair) *commonpb.Ke return nil } -func IsDBEncryptionEnabled(dbProperties []*commonpb.KeyValuePair) bool { +func IsDBEncrypted(dbProperties []*commonpb.KeyValuePair) bool { for _, property := range dbProperties { if property.Key == EncryptionEnabledKey && strings.ToLower(property.Value) == "true" { return true @@ -288,6 +227,111 @@ func IsDBEncryptionEnabled(dbProperties []*commonpb.KeyValuePair) bool { return false } +func RemoveEZByDBProperties(dbProperties []*commonpb.KeyValuePair) error { + ezID, has := ParseEzIDFromProperties(dbProperties) + if !has { + return nil + } + + return RemoveEZ(ezID) +} + +// GetStoragePluginContext returns the local plugin context for RPC from datacoord to datanode +func GetStoragePluginContext(collProps []*commonpb.KeyValuePair, collectionID int64) []*commonpb.KeyValuePair { + if ez := GetEzByCollProperties(collProps, collectionID); ez != nil { + pluginContext, err := GetPluginContext(ez.EzID, ez.CollectionID) + if err != nil { + log.Error("failed to get plugin context", zap.Error(err)) + return nil + } + return pluginContext + } + return nil +} + +func GetReadStoragePluginContext(importEzk string) []*commonpb.KeyValuePair { + readContext, err := ImportEZ(importEzk) + if err != nil { + log.Error("failed to import ezk", zap.Error(err)) + return nil + } + return readContext +} + +// RegisterEZsFromPluginContext registers all EZ contexts from plugin context. +// This processes ALL CipherConfigUnsafeEZK entries (for both read and write contexts). +func RegisterEZsFromPluginContext(context []*commonpb.KeyValuePair) error { + if !IsClusterEncryptionEnabled() { + return nil + } + + for _, value := range context { + if value.GetKey() == CipherConfigUnsafeEZK { + ezID, encryptionKey, err := decodeEZContext(value.GetValue()) + if err != nil { + return err + } + if err := CreateLocalEZ(ezID, encryptionKey); err != nil { + return err + } + } + } + return nil +} + +// GetCPluginContext gets C++ plugin context from the first CipherConfigUnsafeEZK entry. +// Used for creating indexcgopb.StoragePluginContext for C++ segcore. +func GetCPluginContext(context []*commonpb.KeyValuePair, collectionID int64) (*indexcgopb.StoragePluginContext, error) { + if !IsClusterEncryptionEnabled() { + return nil, nil + } + + for _, value := range context { + if value.GetKey() == CipherConfigUnsafeEZK { + ezID, encryptionKey, err := decodeEZContext(value.GetValue()) + if err != nil { + return nil, err + } + + return &indexcgopb.StoragePluginContext{ + CollectionId: collectionID, + EncryptionZoneId: ezID, + EncryptionKey: encryptionKey, + }, nil + } + } + + return nil, nil +} + +func GetCPluginContextByEzID(ezID int64) (*indexcgopb.StoragePluginContext, error) { + if !IsClusterEncryptionEnabled() { + return nil, nil + } + key := GetCipher().GetUnsafeKey(ezID, 0) + if len(key) == 0 { + return nil, errors.Newf("cannot get ez key for ezID=%d", ezID) + } + return &indexcgopb.StoragePluginContext{ + EncryptionZoneId: ezID, + EncryptionKey: base64.StdEncoding.EncodeToString(key), + CollectionId: 0, + }, nil +} + +func BackupEZKFromDBProperties(dbProperties []*commonpb.KeyValuePair) (string, error) { + if !IsDBEncrypted(dbProperties) { + return "", fmt.Errorf("not an encryption zone") + } + + ezID, hasEzID := ParseEzIDFromProperties(dbProperties) + if !hasEzID { + return "", fmt.Errorf("encryption enabled but no ezID found") + } + + return BackupEZ(ezID) +} + // For test only func InitTestCipher() { InitOnceCipher() diff --git a/internal/util/hookutil/cipher_test.go b/internal/util/hookutil/cipher_test.go index 309aa066e2..34468f7c4f 100644 --- a/internal/util/hookutil/cipher_test.go +++ b/internal/util/hookutil/cipher_test.go @@ -20,6 +20,7 @@ import ( "context" "encoding/base64" "fmt" + "strings" "sync" "testing" @@ -135,10 +136,10 @@ func (s *CipherSuite) TestIsDBEncryptionEnabled() { dbProperties := []*commonpb.KeyValuePair{ {Key: EncryptionEnabledKey, Value: "true"}, } - s.True(IsDBEncryptionEnabled(dbProperties)) + s.True(IsDBEncrypted(dbProperties)) dbProperties = []*commonpb.KeyValuePair{} - s.False(IsDBEncryptionEnabled(dbProperties)) + s.False(IsDBEncrypted(dbProperties)) } func (s *CipherSuite) TestTidyDBCipherPropertiesError() { @@ -158,14 +159,14 @@ func (s *CipherSuite) TestTestCipherInit() { s.NoError(err) } -func (s *CipherSuite) TestIsClusterEncyptionEnabled() { +func (s *CipherSuite) TestIsClusterEncryptionEnabled() { // Test when cipher is nil storeCipher(nil) - s.False(IsClusterEncyptionEnabled()) + s.False(IsClusterEncryptionEnabled()) // Test when cipher is not nil InitTestCipher() - s.True(IsClusterEncyptionEnabled()) + s.True(IsClusterEncryptionEnabled()) } func (s *CipherSuite) TestContainsCipherProperty() { @@ -240,11 +241,13 @@ func (s *CipherSuite) TestGetStoragePluginContext() { properties := []*commonpb.KeyValuePair{{Key: EncryptionEzIDKey, Value: "1"}} result = GetStoragePluginContext(properties, 2) s.NotNil(result) - s.Equal(2, len(result)) - s.Equal(CipherConfigCreateEZ, result[0].Key) - s.Equal("1", result[0].Value) - s.Equal(CipherConfigUnsafeEZK, result[1].Key) - s.Equal(base64.StdEncoding.EncodeToString([]byte("unsafe key")), result[1].Value) + s.Equal(1, len(result)) + s.Equal(CipherConfigUnsafeEZK, result[0].Key) + // Verify encoded format: "1:base64Key" + s.Contains(result[0].Value, ":") + parts := strings.Split(result[0].Value, ":") + s.Equal("1", parts[0]) + s.Equal(base64.StdEncoding.EncodeToString([]byte("unsafe key")), parts[1]) result = GetStoragePluginContext([]*commonpb.KeyValuePair{}, 2) s.Nil(result) @@ -277,32 +280,6 @@ func (s *CipherSuite) TestRemoveEZByDBProperties() { s.NoError(err) } -func (s *CipherSuite) TestCreateLocalEZByPluginContext() { - storeCipher(nil) - ctx, err := CreateLocalEZByPluginContext([]*commonpb.KeyValuePair{{Key: CipherConfigCreateEZ, Value: "1"}}) - s.NoError(err) - s.Nil(ctx) - - InitTestCipher() - context := []*commonpb.KeyValuePair{ - {Key: CipherConfigCreateEZ, Value: "123"}, - {Key: CipherConfigUnsafeEZK, Value: "test-key"}, - } - ctx, err = CreateLocalEZByPluginContext(context) - s.NoError(err) - s.NotNil(ctx) - s.Equal(int64(123), ctx.EncryptionZoneId) - s.Equal("test-key", ctx.EncryptionKey) - - ctx, err = CreateLocalEZByPluginContext([]*commonpb.KeyValuePair{{Key: CipherConfigCreateEZ, Value: "invalid"}}) - s.Error(err) - s.Nil(ctx) - - ctx, err = CreateLocalEZByPluginContext([]*commonpb.KeyValuePair{{Key: CipherConfigCreateEZ, Value: "1"}}) - s.NoError(err) - s.Nil(ctx) -} - func (s *CipherSuite) TestCreateEZByDBProperties() { storeCipher(nil) err := CreateEZByDBProperties([]*commonpb.KeyValuePair{{Key: EncryptionEzIDKey, Value: "1"}}) @@ -325,15 +302,15 @@ func (s *CipherSuite) TestCreateEZByDBProperties() { func (s *CipherSuite) TestGetEzPropByDBProperties() { props := []*commonpb.KeyValuePair{{Key: EncryptionEzIDKey, Value: "123"}} - result := GetEzPropByDBProperties(props) + result := getCollEzPropsByDBProps(props) s.NotNil(result) s.Equal(EncryptionEzIDKey, result.Key) s.Equal("123", result.Value) - result = GetEzPropByDBProperties([]*commonpb.KeyValuePair{{Key: "other", Value: "value"}}) + result = getCollEzPropsByDBProps([]*commonpb.KeyValuePair{{Key: "other", Value: "value"}}) s.Nil(result) - result = GetEzPropByDBProperties([]*commonpb.KeyValuePair{}) + result = getCollEzPropsByDBProps([]*commonpb.KeyValuePair{}) s.Nil(result) } @@ -347,13 +324,13 @@ func (s *CipherSuite) TestGetEzByCollPropertiesInvalidValue() { func (s *CipherSuite) TestIsDBEncryptionEnabledCaseInsensitive() { props := []*commonpb.KeyValuePair{{Key: EncryptionEnabledKey, Value: "True"}} - s.True(IsDBEncryptionEnabled(props)) + s.True(IsDBEncrypted(props)) props = []*commonpb.KeyValuePair{{Key: EncryptionEnabledKey, Value: "TRUE"}} - s.True(IsDBEncryptionEnabled(props)) + s.True(IsDBEncrypted(props)) props = []*commonpb.KeyValuePair{{Key: EncryptionEnabledKey, Value: "false"}} - s.False(IsDBEncryptionEnabled(props)) + s.False(IsDBEncrypted(props)) } func (s *CipherSuite) TestRegisterCallback() { @@ -409,3 +386,155 @@ func (s *CipherSuite) TestGetEzByCollPropertiesNotFound() { result := GetEzByCollProperties(props, 456) s.Nil(result) } + +func (s *CipherSuite) TestBackupEZ() { + storeCipher(nil) + _, err := BackupEZ(123) + s.Error(err) + s.Equal(ErrCipherPluginMissing, err) + + InitTestCipher() + _, err = BackupEZ(123) + s.Error(err) + s.Contains(err.Error(), "does not support backup operation") +} + +func (s *CipherSuite) TestBackupEZKFromDBProperties() { + storeCipher(nil) + props := []*commonpb.KeyValuePair{ + {Key: EncryptionEnabledKey, Value: "true"}, + {Key: EncryptionEzIDKey, Value: "123"}, + } + _, err := BackupEZKFromDBProperties(props) + s.Error(err) + s.Equal(ErrCipherPluginMissing, err) + + props = []*commonpb.KeyValuePair{ + {Key: EncryptionEnabledKey, Value: "false"}, + } + _, err = BackupEZKFromDBProperties(props) + s.Error(err) + s.Contains(err.Error(), "not an encryption zone") + + props = []*commonpb.KeyValuePair{ + {Key: EncryptionEnabledKey, Value: "true"}, + } + _, err = BackupEZKFromDBProperties(props) + s.Error(err) + s.Contains(err.Error(), "no ezID found") +} + +func (s *CipherSuite) TestImportEZ() { + storeCipher(nil) + result, err := ImportEZ("") + s.NoError(err) + s.Nil(result) + + result, err = ImportEZ("some-base64-string") + s.NoError(err) + s.Nil(result) + + InitTestCipher() + result, err = ImportEZ("invalid-base64") + s.Error(err) + s.Nil(result) + s.Contains(err.Error(), "failed to decode EZK") + + result, err = ImportEZ(base64.StdEncoding.EncodeToString([]byte("not-json"))) + s.Error(err) + s.Nil(result) + s.Contains(err.Error(), "failed to unmarshal EZK") + + ezkJSON := `{"ez_id":123}` + ezkBase64 := base64.StdEncoding.EncodeToString([]byte(ezkJSON)) + result, err = ImportEZ(ezkBase64) + s.NoError(err) + s.NotNil(result) + s.Equal(1, len(result)) + s.Equal(CipherConfigUnsafeEZK, result[0].Key) + // Verify encoded format: "123:base64Key" + s.Contains(result[0].Value, ":") + parts := strings.Split(result[0].Value, ":") + s.Equal("123", parts[0]) + s.Equal(base64.StdEncoding.EncodeToString([]byte("unsafe key")), parts[1]) +} + +func (s *CipherSuite) TestIsEncryptionEnabled() { + storeCipher(nil) + s.False(IsClusterEncryptionEnabled()) + + InitTestCipher() + s.True(IsClusterEncryptionEnabled()) +} + +func (s *CipherSuite) TestParseEzIDFromProperties() { + props := []*commonpb.KeyValuePair{ + {Key: EncryptionEzIDKey, Value: "123"}, + } + ezID, found := ParseEzIDFromProperties(props) + s.True(found) + s.Equal(int64(123), ezID) + + props = []*commonpb.KeyValuePair{ + {Key: "other_key", Value: "value"}, + } + ezID, found = ParseEzIDFromProperties(props) + s.False(found) + s.Equal(int64(0), ezID) + + props = []*commonpb.KeyValuePair{ + {Key: EncryptionEzIDKey, Value: "invalid"}, + } + ezID, found = ParseEzIDFromProperties(props) + s.False(found) + s.Equal(int64(0), ezID) +} + +func (s *CipherSuite) TestCreateEZ() { + storeCipher(nil) + err := CreateEZ(123, "test-kms-key") + s.NoError(err) + + InitTestCipher() + err = CreateEZ(456, "test-kms-key") + s.NoError(err) +} + +func (s *CipherSuite) TestCreateLocalEZ() { + storeCipher(nil) + err := CreateLocalEZ(123, "test-encryption-key") + s.NoError(err) + + InitTestCipher() + err = CreateLocalEZ(456, "test-encryption-key") + s.NoError(err) +} + +func (s *CipherSuite) TestRemoveEZ() { + storeCipher(nil) + err := RemoveEZ(123) + s.NoError(err) + + InitTestCipher() + err = RemoveEZ(456) + s.NoError(err) +} + +func (s *CipherSuite) TestGetPluginContext() { + storeCipher(nil) + result, err := GetPluginContext(123, 456) + s.NoError(err) + s.Nil(result) + + InitTestCipher() + result, err = GetPluginContext(123, 456) + s.NoError(err) + s.NotNil(result) + s.Equal(1, len(result)) + s.Equal(CipherConfigUnsafeEZK, result[0].Key) + // Verify encoded format: "123:base64Key" + s.Contains(result[0].Value, ":") + parts := strings.Split(result[0].Value, ":") + s.Equal("123", parts[0]) + s.Equal(base64.StdEncoding.EncodeToString([]byte("unsafe key")), parts[1]) +} diff --git a/internal/util/hookutil/ez.go b/internal/util/hookutil/ez.go new file mode 100644 index 0000000000..55b5a0ec1b --- /dev/null +++ b/internal/util/hookutil/ez.go @@ -0,0 +1,178 @@ +// Licensed to the LF AI & Data foundation under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hookutil + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "strconv" + "strings" + + "github.com/cockroachdb/errors" + + "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" +) + +const ( + // Used in Plugins + CipherConfigCreateEZ = "cipher.ez.create" + CipherConfigRemoveEZ = "cipher.ez.remove" + CipherConfigImportEZ = "cipher.ez.import" + CipherConfigMilvusRoleName = "cipher.milvusRoleName" + CipherConfigKeyKmsKeyArn = "cipher.kmsKeyArn" + CipherConfigUnsafeEZK = "cipher.ezk" +) + +func encodeEZContext(ezID int64, key []byte) string { + return fmt.Sprintf("%d:%s", ezID, base64.StdEncoding.EncodeToString(key)) +} + +func decodeEZContext(encoded string) (ezID int64, key string, err error) { + parts := strings.Split(encoded, ":") + if len(parts) != 2 { + return 0, "", fmt.Errorf("invalid EZ context format: %s", encoded) + } + + ezID, err = strconv.ParseInt(parts[0], 10, 64) + if err != nil { + return 0, "", fmt.Errorf("invalid ezID in context: %w", err) + } + + return ezID, parts[1], nil +} + +func ParseEzIDFromProperties(properties []*commonpb.KeyValuePair) (int64, bool) { + for _, property := range properties { + if property.Key == EncryptionEzIDKey { + ezID, err := strconv.ParseInt(property.Value, 10, 64) + if err != nil { + return 0, false + } + return ezID, true + } + } + return 0, false +} + +func CreateEZ(ezID int64, kmsKeyArn string) error { + cipher := GetCipher() + if cipher == nil { + return nil + } + + config := map[string]string{ + CipherConfigCreateEZ: strconv.FormatInt(ezID, 10), + CipherConfigKeyKmsKeyArn: kmsKeyArn, + } + + return cipher.Init(config) +} + +func CreateLocalEZ(ezID int64, encryptionKey string) error { + cipher := GetCipher() + if cipher == nil { + return nil + } + + config := map[string]string{ + CipherConfigCreateEZ: strconv.FormatInt(ezID, 10), + CipherConfigUnsafeEZK: encryptionKey, + } + + return cipher.Init(config) +} + +func RemoveEZ(ezID int64) error { + cipher := GetCipher() + if cipher == nil { + return nil + } + + config := map[string]string{ + CipherConfigRemoveEZ: strconv.FormatInt(ezID, 10), + } + + return cipher.Init(config) +} + +func BackupEZ(ezID int64) (string, error) { + cipher := GetCipher() + if cipher == nil { + return "", ErrCipherPluginMissing + } + + backupProvider, ok := cipher.(BackupInterface) + if !ok { + return "", fmt.Errorf("cipher plugin does not support backup operation") + } + + return backupProvider.Backup(ezID) +} + +func GetPluginContext(ezID int64, collectionID int64) ([]*commonpb.KeyValuePair, error) { + cipher := GetCipher() + if cipher == nil { + return nil, nil + } + + key := cipher.GetUnsafeKey(ezID, collectionID) + if len(key) == 0 { + return nil, errors.Newf("cannot get ez key for ezID=%d, collectionID=%d", ezID, collectionID) + } + + return []*commonpb.KeyValuePair{{ + Key: CipherConfigUnsafeEZK, + Value: encodeEZContext(ezID, key), + }}, nil +} + +func ImportEZ(importEzk string) ([]*commonpb.KeyValuePair, error) { + cipher := GetCipher() + if cipher == nil || importEzk == "" { + return nil, nil + } + + config := map[string]string{ + CipherConfigImportEZ: importEzk, + } + if err := cipher.Init(config); err != nil { + return nil, fmt.Errorf("failed to import EZK: %w", err) + } + + ezID, err := GetEzIDByImportEzk(importEzk) + if err != nil { + return nil, err + } + return GetPluginContext(ezID, 0) +} + +func GetEzIDByImportEzk(importEzk string) (int64, error) { + ezkBytes, err := base64.StdEncoding.DecodeString(importEzk) + if err != nil { + return 0, fmt.Errorf("failed to decode EZK: %w", err) + } + + type EZKData struct { + EzID int64 `json:"ez_id"` + } + var ezkData EZKData + if err := json.Unmarshal(ezkBytes, &ezkData); err != nil { + return 0, fmt.Errorf("failed to unmarshal EZK: %w", err) + } + return ezkData.EzID, nil +} diff --git a/internal/util/importutilv2/binlog/reader.go b/internal/util/importutilv2/binlog/reader.go index 35990344c6..e7ba16c9a1 100644 --- a/internal/util/importutilv2/binlog/reader.go +++ b/internal/util/importutilv2/binlog/reader.go @@ -28,6 +28,7 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/internal/storage" + "github.com/milvus-io/milvus/internal/util/hookutil" "github.com/milvus-io/milvus/pkg/v2/log" "github.com/milvus-io/milvus/pkg/v2/proto/indexpb" "github.com/milvus-io/milvus/pkg/v2/util/merr" @@ -58,6 +59,7 @@ func NewReader(ctx context.Context, tsStart, tsEnd uint64, bufferSize int, + importEz string, ) (*reader, error) { systemFieldsAbsent := true for _, field := range schema.Fields { @@ -77,14 +79,14 @@ func NewReader(ctx context.Context, fileSize: atomic.NewInt64(0), bufferSize: bufferSize, } - err := r.init(paths, tsStart, tsEnd, storageConfig) + err := r.init(paths, tsStart, tsEnd, storageConfig, importEz) if err != nil { return nil, err } return r, nil } -func (r *reader) init(paths []string, tsStart, tsEnd uint64, storageConfig *indexpb.StorageConfig) error { +func (r *reader) init(paths []string, tsStart, tsEnd uint64, storageConfig *indexpb.StorageConfig, importEZ string) error { if tsStart != 0 || tsEnd != math.MaxUint64 { r.filters = append(r.filters, FilterWithTimeRange(tsStart, tsEnd)) } @@ -113,17 +115,27 @@ func (r *reader) init(paths []string, tsStart, tsEnd uint64, storageConfig *inde validIDs := lo.Keys(r.insertLogs) log.Info("create binlog reader for these fields", zap.Any("validIDs", validIDs)) - // TODO:[GOOSE] Backup related changes: No CollectionID and schema comes from to write collection - // means this reader cannot read encrypted files. - // StoragePlugin config is wrong for backuped binlogs - rr, err := storage.NewBinlogRecordReader(r.ctx, binlogs, r.schema, + rwOptions := []storage.RwOption{ storage.WithVersion(r.storageVersion), - storage.WithBufferSize(32*1024*1024), + storage.WithBufferSize(32 * 1024 * 1024), storage.WithDownloader(func(ctx context.Context, paths []string) ([][]byte, error) { return r.cm.MultiRead(ctx, paths) }), storage.WithStorageConfig(storageConfig), - ) + } + + if len(importEZ) > 0 { + ezID, err := hookutil.GetEzIDByImportEzk(importEZ) + if err != nil { + return err + } + pluginContext, err := hookutil.GetCPluginContextByEzID(ezID) + if err != nil { + return err + } + rwOptions = append(rwOptions, storage.WithPluginContext(pluginContext)) + } + rr, err := storage.NewBinlogRecordReader(r.ctx, binlogs, r.schema, rwOptions...) if err != nil { return err } diff --git a/internal/util/importutilv2/binlog/reader_test.go b/internal/util/importutilv2/binlog/reader_test.go index 6c0701c277..dde872823f 100644 --- a/internal/util/importutilv2/binlog/reader_test.go +++ b/internal/util/importutilv2/binlog/reader_test.go @@ -386,7 +386,7 @@ func (suite *ReaderSuite) run(dataType schemapb.DataType, elemType schemapb.Data cm, originalInsertData := suite.createMockChunk(schema, insertBinlogs, true) cm.EXPECT().Size(mock.Anything, mock.Anything).Return(128, nil) - reader, err := NewReader(context.Background(), cm, schema, &indexpb.StorageConfig{}, storage.StorageV1, []string{insertPrefix, deltaPrefix}, suite.tsStart, suite.tsEnd, 64*1024*1024) + reader, err := NewReader(context.Background(), cm, schema, &indexpb.StorageConfig{}, storage.StorageV1, []string{insertPrefix, deltaPrefix}, suite.tsStart, suite.tsEnd, 64*1024*1024, "") suite.NoError(err) insertData, err := reader.Read() suite.NoError(err) @@ -585,18 +585,18 @@ func (suite *ReaderSuite) TestVerify() { checkFunc := func() { cm, _ := suite.createMockChunk(schema, insertBinlogs, false) - reader, err := NewReader(context.Background(), cm, schema, &indexpb.StorageConfig{}, storage.StorageV1, []string{insertPrefix, deltaPrefix}, suite.tsStart, suite.tsEnd, 64*1024*1024) + reader, err := NewReader(context.Background(), cm, schema, &indexpb.StorageConfig{}, storage.StorageV1, []string{insertPrefix, deltaPrefix}, suite.tsStart, suite.tsEnd, 64*1024*1024, "") suite.Error(err) suite.Nil(reader) } // no insert binlogs to import - reader, err := NewReader(context.Background(), nil, schema, &indexpb.StorageConfig{}, storage.StorageV1, []string{}, suite.tsStart, suite.tsEnd, 64*1024*1024) + reader, err := NewReader(context.Background(), nil, schema, &indexpb.StorageConfig{}, storage.StorageV1, []string{}, suite.tsStart, suite.tsEnd, 64*1024*1024, "") suite.Error(err) suite.Nil(reader) // too many input paths - reader, err = NewReader(context.Background(), nil, schema, &indexpb.StorageConfig{}, storage.StorageV1, []string{insertPrefix, deltaPrefix, "dummy"}, suite.tsStart, suite.tsEnd, 64*1024*1024) + reader, err = NewReader(context.Background(), nil, schema, &indexpb.StorageConfig{}, storage.StorageV1, []string{insertPrefix, deltaPrefix, "dummy"}, suite.tsStart, suite.tsEnd, 64*1024*1024, "") suite.Error(err) suite.Nil(reader) @@ -749,7 +749,7 @@ func (suite *ReaderSuite) TestZeroDeltaRead() { checkFunc := func(targetSchema *schemapb.CollectionSchema, expectReadBinlogs map[int64][]string) { cm := mockChunkFunc(sourceSchema, expectReadBinlogs) - reader, err := NewReader(context.Background(), cm, targetSchema, &indexpb.StorageConfig{}, storage.StorageV1, []string{insertPrefix, deltaPrefix}, suite.tsStart, suite.tsEnd, 64*1024*1024) + reader, err := NewReader(context.Background(), cm, targetSchema, &indexpb.StorageConfig{}, storage.StorageV1, []string{insertPrefix, deltaPrefix}, suite.tsStart, suite.tsEnd, 64*1024*1024, "") suite.NoError(err) suite.NotNil(reader) diff --git a/internal/util/importutilv2/option.go b/internal/util/importutilv2/option.go index d89159dd85..33c2f51fcb 100644 --- a/internal/util/importutilv2/option.go +++ b/internal/util/importutilv2/option.go @@ -65,6 +65,9 @@ const ( StartTs2 = "startTs" EndTs = "end_ts" EndTs2 = "endTs" + + // EZK is the base64-encoded encryption zone key for reading encrypted backup data. + EZK = "ezk" ) type Options []*commonpb.KeyValuePair @@ -183,3 +186,11 @@ func GetCSVNullKey(options Options) (string, error) { } return nullKey, nil } + +func GetEZK(options Options) (string, error) { + ezk, err := funcutil.GetAttrByKeyFromRepeatedKV(EZK, options) + if err != nil || len(ezk) == 0 { + return "", nil + } + return ezk, nil +} diff --git a/internal/util/importutilv2/reader.go b/internal/util/importutilv2/reader.go index 3c72b8f558..ff694ffd65 100644 --- a/internal/util/importutilv2/reader.go +++ b/internal/util/importutilv2/reader.go @@ -63,7 +63,8 @@ func NewReader(ctx context.Context, if err != nil { return nil, err } - return binlog.NewReader(ctx, cm, schema, storageConfig, storageVersion, paths, tsStart, tsEnd, bufferSize) + importEz, _ := GetEZK(options) + return binlog.NewReader(ctx, cm, schema, storageConfig, storageVersion, paths, tsStart, tsEnd, bufferSize, importEz) } fileType, err := GetFileType(importFile) diff --git a/internal/util/initcore/init_core.go b/internal/util/initcore/init_core.go index dc76af20b5..fa0d2f6f07 100644 --- a/internal/util/initcore/init_core.go +++ b/internal/util/initcore/init_core.go @@ -641,7 +641,7 @@ func serializeHeaders(headerstr string) string { } func InitPluginLoader() error { - if hookutil.IsClusterEncyptionEnabled() { + if hookutil.IsClusterEncryptionEnabled() { cSoPath := C.CString(paramtable.GetCipherParams().SoPathCpp.GetValue()) log.Info("Init PluginLoader", zap.String("soPath", paramtable.GetCipherParams().SoPathCpp.GetValue())) defer C.free(unsafe.Pointer(cSoPath)) diff --git a/internal/util/mock/grpc_rootcoord_client.go b/internal/util/mock/grpc_rootcoord_client.go index df07f2993d..df8fb327b1 100644 --- a/internal/util/mock/grpc_rootcoord_client.go +++ b/internal/util/mock/grpc_rootcoord_client.go @@ -325,3 +325,7 @@ func (m *GrpcRootCoordClient) GetQuotaMetrics(ctx context.Context, in *internalp func (m *GrpcRootCoordClient) TruncateCollection(ctx context.Context, in *milvuspb.TruncateCollectionRequest, opts ...grpc.CallOption) (*milvuspb.TruncateCollectionResponse, error) { return &milvuspb.TruncateCollectionResponse{}, m.Err } + +func (m *GrpcRootCoordClient) BackupEzk(ctx context.Context, in *internalpb.BackupEzkRequest, opts ...grpc.CallOption) (*internalpb.BackupEzkResponse, error) { + return &internalpb.BackupEzkResponse{}, m.Err +} diff --git a/pkg/proto/internal.proto b/pkg/proto/internal.proto index ca2e38a5c5..168cfecd8b 100644 --- a/pkg/proto/internal.proto +++ b/pkg/proto/internal.proto @@ -459,4 +459,14 @@ message FileResourceInfo { message SyncFileResourceRequest{ repeated FileResourceInfo resources = 1; uint64 version = 2; -} \ No newline at end of file +} + +message BackupEzkRequest { + common.MsgBase base = 1; + string db_name = 2; +} + +message BackupEzkResponse { + common.Status status = 1; + string ezk = 2; +} diff --git a/pkg/proto/internalpb/internal.pb.go b/pkg/proto/internalpb/internal.pb.go index c2495ac242..3f406b38f4 100644 --- a/pkg/proto/internalpb/internal.pb.go +++ b/pkg/proto/internalpb/internal.pb.go @@ -4156,6 +4156,116 @@ func (x *SyncFileResourceRequest) GetVersion() uint64 { return 0 } +type BackupEzkRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Base *commonpb.MsgBase `protobuf:"bytes,1,opt,name=base,proto3" json:"base,omitempty"` + DbName string `protobuf:"bytes,2,opt,name=db_name,json=dbName,proto3" json:"db_name,omitempty"` +} + +func (x *BackupEzkRequest) Reset() { + *x = BackupEzkRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_internal_proto_msgTypes[48] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *BackupEzkRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BackupEzkRequest) ProtoMessage() {} + +func (x *BackupEzkRequest) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[48] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BackupEzkRequest.ProtoReflect.Descriptor instead. +func (*BackupEzkRequest) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{48} +} + +func (x *BackupEzkRequest) GetBase() *commonpb.MsgBase { + if x != nil { + return x.Base + } + return nil +} + +func (x *BackupEzkRequest) GetDbName() string { + if x != nil { + return x.DbName + } + return "" +} + +type BackupEzkResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Status *commonpb.Status `protobuf:"bytes,1,opt,name=status,proto3" json:"status,omitempty"` + Ezk string `protobuf:"bytes,2,opt,name=ezk,proto3" json:"ezk,omitempty"` +} + +func (x *BackupEzkResponse) Reset() { + *x = BackupEzkResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_internal_proto_msgTypes[49] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *BackupEzkResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*BackupEzkResponse) ProtoMessage() {} + +func (x *BackupEzkResponse) ProtoReflect() protoreflect.Message { + mi := &file_internal_proto_msgTypes[49] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use BackupEzkResponse.ProtoReflect.Descriptor instead. +func (*BackupEzkResponse) Descriptor() ([]byte, []int) { + return file_internal_proto_rawDescGZIP(), []int{49} +} + +func (x *BackupEzkResponse) GetStatus() *commonpb.Status { + if x != nil { + return x.Status + } + return nil +} + +func (x *BackupEzkResponse) GetEzk() string { + if x != nil { + return x.Ezk + } + return "" +} + var File_internal_proto protoreflect.FileDescriptor var file_internal_proto_rawDesc = []byte{ @@ -4877,36 +4987,48 @@ var file_internal_proto_rawDesc = []byte{ 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x49, 0x6e, 0x66, 0x6f, 0x52, 0x09, 0x72, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x04, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x2a, 0x45, 0x0a, 0x09, - 0x52, 0x61, 0x74, 0x65, 0x53, 0x63, 0x6f, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x6c, 0x75, - 0x73, 0x74, 0x65, 0x72, 0x10, 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x61, 0x74, 0x61, 0x62, 0x61, - 0x73, 0x65, 0x10, 0x01, 0x12, 0x0e, 0x0a, 0x0a, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, - 0x6e, 0x10, 0x03, 0x2a, 0xc4, 0x01, 0x0a, 0x08, 0x52, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, - 0x12, 0x11, 0x0a, 0x0d, 0x44, 0x44, 0x4c, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, - 0x6e, 0x10, 0x00, 0x12, 0x10, 0x0a, 0x0c, 0x44, 0x44, 0x4c, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, - 0x69, 0x6f, 0x6e, 0x10, 0x01, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x44, 0x4c, 0x49, 0x6e, 0x64, 0x65, - 0x78, 0x10, 0x02, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x44, 0x4c, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x10, - 0x03, 0x12, 0x11, 0x0a, 0x0d, 0x44, 0x44, 0x4c, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x69, - 0x6f, 0x6e, 0x10, 0x04, 0x12, 0x0d, 0x0a, 0x09, 0x44, 0x4d, 0x4c, 0x49, 0x6e, 0x73, 0x65, 0x72, - 0x74, 0x10, 0x05, 0x12, 0x0d, 0x0a, 0x09, 0x44, 0x4d, 0x4c, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, - 0x10, 0x06, 0x12, 0x0f, 0x0a, 0x0b, 0x44, 0x4d, 0x4c, 0x42, 0x75, 0x6c, 0x6b, 0x4c, 0x6f, 0x61, - 0x64, 0x10, 0x07, 0x12, 0x0d, 0x0a, 0x09, 0x44, 0x51, 0x4c, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, - 0x10, 0x08, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x51, 0x4c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x10, 0x09, - 0x12, 0x0d, 0x0a, 0x09, 0x44, 0x4d, 0x4c, 0x55, 0x70, 0x73, 0x65, 0x72, 0x74, 0x10, 0x0a, 0x12, - 0x09, 0x0a, 0x05, 0x44, 0x44, 0x4c, 0x44, 0x42, 0x10, 0x0b, 0x2a, 0x83, 0x01, 0x0a, 0x0e, 0x49, - 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x08, 0x0a, - 0x04, 0x4e, 0x6f, 0x6e, 0x65, 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x65, 0x6e, 0x64, 0x69, - 0x6e, 0x67, 0x10, 0x01, 0x12, 0x10, 0x0a, 0x0c, 0x50, 0x72, 0x65, 0x49, 0x6d, 0x70, 0x6f, 0x72, - 0x74, 0x69, 0x6e, 0x67, 0x10, 0x02, 0x12, 0x0d, 0x0a, 0x09, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, - 0x69, 0x6e, 0x67, 0x10, 0x03, 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x10, - 0x04, 0x12, 0x0d, 0x0a, 0x09, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x10, 0x05, - 0x12, 0x11, 0x0a, 0x0d, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, - 0x67, 0x10, 0x06, 0x12, 0x0b, 0x0a, 0x07, 0x53, 0x6f, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x10, 0x07, - 0x42, 0x35, 0x5a, 0x33, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, - 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2d, 0x69, 0x6f, 0x2f, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2f, - 0x70, 0x6b, 0x67, 0x2f, 0x76, 0x32, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x69, 0x6e, 0x74, - 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x01, 0x28, 0x04, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x22, 0x5d, 0x0a, 0x10, + 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x45, 0x7a, 0x6b, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, + 0x12, 0x30, 0x0a, 0x04, 0x62, 0x61, 0x73, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1c, + 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x63, 0x6f, + 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x4d, 0x73, 0x67, 0x42, 0x61, 0x73, 0x65, 0x52, 0x04, 0x62, 0x61, + 0x73, 0x65, 0x12, 0x17, 0x0a, 0x07, 0x64, 0x62, 0x5f, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x06, 0x64, 0x62, 0x4e, 0x61, 0x6d, 0x65, 0x22, 0x5a, 0x0a, 0x11, 0x42, + 0x61, 0x63, 0x6b, 0x75, 0x70, 0x45, 0x7a, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, + 0x12, 0x33, 0x0a, 0x06, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, + 0x32, 0x1b, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, + 0x63, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x2e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x52, 0x06, 0x73, + 0x74, 0x61, 0x74, 0x75, 0x73, 0x12, 0x10, 0x0a, 0x03, 0x65, 0x7a, 0x6b, 0x18, 0x02, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x03, 0x65, 0x7a, 0x6b, 0x2a, 0x45, 0x0a, 0x09, 0x52, 0x61, 0x74, 0x65, 0x53, + 0x63, 0x6f, 0x70, 0x65, 0x12, 0x0b, 0x0a, 0x07, 0x43, 0x6c, 0x75, 0x73, 0x74, 0x65, 0x72, 0x10, + 0x00, 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x10, 0x01, 0x12, + 0x0e, 0x0a, 0x0a, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x10, 0x02, 0x12, + 0x0d, 0x0a, 0x09, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x10, 0x03, 0x2a, 0xc4, + 0x01, 0x0a, 0x08, 0x52, 0x61, 0x74, 0x65, 0x54, 0x79, 0x70, 0x65, 0x12, 0x11, 0x0a, 0x0d, 0x44, + 0x44, 0x4c, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x10, 0x00, 0x12, 0x10, + 0x0a, 0x0c, 0x44, 0x44, 0x4c, 0x50, 0x61, 0x72, 0x74, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x10, 0x01, + 0x12, 0x0c, 0x0a, 0x08, 0x44, 0x44, 0x4c, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x10, 0x02, 0x12, 0x0c, + 0x0a, 0x08, 0x44, 0x44, 0x4c, 0x46, 0x6c, 0x75, 0x73, 0x68, 0x10, 0x03, 0x12, 0x11, 0x0a, 0x0d, + 0x44, 0x44, 0x4c, 0x43, 0x6f, 0x6d, 0x70, 0x61, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x10, 0x04, 0x12, + 0x0d, 0x0a, 0x09, 0x44, 0x4d, 0x4c, 0x49, 0x6e, 0x73, 0x65, 0x72, 0x74, 0x10, 0x05, 0x12, 0x0d, + 0x0a, 0x09, 0x44, 0x4d, 0x4c, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x10, 0x06, 0x12, 0x0f, 0x0a, + 0x0b, 0x44, 0x4d, 0x4c, 0x42, 0x75, 0x6c, 0x6b, 0x4c, 0x6f, 0x61, 0x64, 0x10, 0x07, 0x12, 0x0d, + 0x0a, 0x09, 0x44, 0x51, 0x4c, 0x53, 0x65, 0x61, 0x72, 0x63, 0x68, 0x10, 0x08, 0x12, 0x0c, 0x0a, + 0x08, 0x44, 0x51, 0x4c, 0x51, 0x75, 0x65, 0x72, 0x79, 0x10, 0x09, 0x12, 0x0d, 0x0a, 0x09, 0x44, + 0x4d, 0x4c, 0x55, 0x70, 0x73, 0x65, 0x72, 0x74, 0x10, 0x0a, 0x12, 0x09, 0x0a, 0x05, 0x44, 0x44, + 0x4c, 0x44, 0x42, 0x10, 0x0b, 0x2a, 0x83, 0x01, 0x0a, 0x0e, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, + 0x4a, 0x6f, 0x62, 0x53, 0x74, 0x61, 0x74, 0x65, 0x12, 0x08, 0x0a, 0x04, 0x4e, 0x6f, 0x6e, 0x65, + 0x10, 0x00, 0x12, 0x0b, 0x0a, 0x07, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x10, 0x01, 0x12, + 0x10, 0x0a, 0x0c, 0x50, 0x72, 0x65, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x10, + 0x02, 0x12, 0x0d, 0x0a, 0x09, 0x49, 0x6d, 0x70, 0x6f, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x10, 0x03, + 0x12, 0x0a, 0x0a, 0x06, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x10, 0x04, 0x12, 0x0d, 0x0a, 0x09, + 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64, 0x10, 0x05, 0x12, 0x11, 0x0a, 0x0d, 0x49, + 0x6e, 0x64, 0x65, 0x78, 0x42, 0x75, 0x69, 0x6c, 0x64, 0x69, 0x6e, 0x67, 0x10, 0x06, 0x12, 0x0b, + 0x0a, 0x07, 0x53, 0x6f, 0x72, 0x74, 0x69, 0x6e, 0x67, 0x10, 0x07, 0x42, 0x35, 0x5a, 0x33, 0x67, + 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, + 0x2d, 0x69, 0x6f, 0x2f, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2f, 0x70, 0x6b, 0x67, 0x2f, 0x76, + 0x32, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, + 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -4922,7 +5044,7 @@ func file_internal_proto_rawDescGZIP() []byte { } var file_internal_proto_enumTypes = make([]protoimpl.EnumInfo, 3) -var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 49) +var file_internal_proto_msgTypes = make([]protoimpl.MessageInfo, 51) var file_internal_proto_goTypes = []interface{}{ (RateScope)(0), // 0: milvus.proto.internal.RateScope (RateType)(0), // 1: milvus.proto.internal.RateType @@ -4975,88 +5097,92 @@ var file_internal_proto_goTypes = []interface{}{ (*GetQuotaMetricsResponse)(nil), // 48: milvus.proto.internal.GetQuotaMetricsResponse (*FileResourceInfo)(nil), // 49: milvus.proto.internal.FileResourceInfo (*SyncFileResourceRequest)(nil), // 50: milvus.proto.internal.SyncFileResourceRequest - nil, // 51: milvus.proto.internal.SearchResults.ChannelsMvccEntry - (*commonpb.Address)(nil), // 52: milvus.proto.common.Address - (*commonpb.KeyValuePair)(nil), // 53: milvus.proto.common.KeyValuePair - (*commonpb.Status)(nil), // 54: milvus.proto.common.Status - (*commonpb.MsgBase)(nil), // 55: milvus.proto.common.MsgBase - (commonpb.DslType)(0), // 56: milvus.proto.common.DslType - (commonpb.ConsistencyLevel)(0), // 57: milvus.proto.common.ConsistencyLevel - (*schemapb.IDs)(nil), // 58: milvus.proto.schema.IDs - (*schemapb.FieldData)(nil), // 59: milvus.proto.schema.FieldData - (*milvuspb.PrivilegeGroupInfo)(nil), // 60: milvus.proto.milvus.PrivilegeGroupInfo - (*schemapb.CollectionSchema)(nil), // 61: milvus.proto.schema.CollectionSchema - (commonpb.SegmentState)(0), // 62: milvus.proto.common.SegmentState - (commonpb.SegmentLevel)(0), // 63: milvus.proto.common.SegmentLevel + (*BackupEzkRequest)(nil), // 51: milvus.proto.internal.BackupEzkRequest + (*BackupEzkResponse)(nil), // 52: milvus.proto.internal.BackupEzkResponse + nil, // 53: milvus.proto.internal.SearchResults.ChannelsMvccEntry + (*commonpb.Address)(nil), // 54: milvus.proto.common.Address + (*commonpb.KeyValuePair)(nil), // 55: milvus.proto.common.KeyValuePair + (*commonpb.Status)(nil), // 56: milvus.proto.common.Status + (*commonpb.MsgBase)(nil), // 57: milvus.proto.common.MsgBase + (commonpb.DslType)(0), // 58: milvus.proto.common.DslType + (commonpb.ConsistencyLevel)(0), // 59: milvus.proto.common.ConsistencyLevel + (*schemapb.IDs)(nil), // 60: milvus.proto.schema.IDs + (*schemapb.FieldData)(nil), // 61: milvus.proto.schema.FieldData + (*milvuspb.PrivilegeGroupInfo)(nil), // 62: milvus.proto.milvus.PrivilegeGroupInfo + (*schemapb.CollectionSchema)(nil), // 63: milvus.proto.schema.CollectionSchema + (commonpb.SegmentState)(0), // 64: milvus.proto.common.SegmentState + (commonpb.SegmentLevel)(0), // 65: milvus.proto.common.SegmentLevel } var file_internal_proto_depIdxs = []int32{ - 52, // 0: milvus.proto.internal.NodeInfo.address:type_name -> milvus.proto.common.Address - 53, // 1: milvus.proto.internal.InitParams.start_params:type_name -> milvus.proto.common.KeyValuePair - 54, // 2: milvus.proto.internal.StringList.status:type_name -> milvus.proto.common.Status - 55, // 3: milvus.proto.internal.GetStatisticsRequest.base:type_name -> milvus.proto.common.MsgBase - 55, // 4: milvus.proto.internal.GetStatisticsResponse.base:type_name -> milvus.proto.common.MsgBase - 54, // 5: milvus.proto.internal.GetStatisticsResponse.status:type_name -> milvus.proto.common.Status - 53, // 6: milvus.proto.internal.GetStatisticsResponse.stats:type_name -> milvus.proto.common.KeyValuePair - 55, // 7: milvus.proto.internal.CreateAliasRequest.base:type_name -> milvus.proto.common.MsgBase - 55, // 8: milvus.proto.internal.DropAliasRequest.base:type_name -> milvus.proto.common.MsgBase - 55, // 9: milvus.proto.internal.AlterAliasRequest.base:type_name -> milvus.proto.common.MsgBase - 55, // 10: milvus.proto.internal.CreateIndexRequest.base:type_name -> milvus.proto.common.MsgBase - 53, // 11: milvus.proto.internal.CreateIndexRequest.extra_params:type_name -> milvus.proto.common.KeyValuePair - 56, // 12: milvus.proto.internal.SubSearchRequest.dsl_type:type_name -> milvus.proto.common.DslType - 55, // 13: milvus.proto.internal.SearchRequest.base:type_name -> milvus.proto.common.MsgBase - 56, // 14: milvus.proto.internal.SearchRequest.dsl_type:type_name -> milvus.proto.common.DslType + 54, // 0: milvus.proto.internal.NodeInfo.address:type_name -> milvus.proto.common.Address + 55, // 1: milvus.proto.internal.InitParams.start_params:type_name -> milvus.proto.common.KeyValuePair + 56, // 2: milvus.proto.internal.StringList.status:type_name -> milvus.proto.common.Status + 57, // 3: milvus.proto.internal.GetStatisticsRequest.base:type_name -> milvus.proto.common.MsgBase + 57, // 4: milvus.proto.internal.GetStatisticsResponse.base:type_name -> milvus.proto.common.MsgBase + 56, // 5: milvus.proto.internal.GetStatisticsResponse.status:type_name -> milvus.proto.common.Status + 55, // 6: milvus.proto.internal.GetStatisticsResponse.stats:type_name -> milvus.proto.common.KeyValuePair + 57, // 7: milvus.proto.internal.CreateAliasRequest.base:type_name -> milvus.proto.common.MsgBase + 57, // 8: milvus.proto.internal.DropAliasRequest.base:type_name -> milvus.proto.common.MsgBase + 57, // 9: milvus.proto.internal.AlterAliasRequest.base:type_name -> milvus.proto.common.MsgBase + 57, // 10: milvus.proto.internal.CreateIndexRequest.base:type_name -> milvus.proto.common.MsgBase + 55, // 11: milvus.proto.internal.CreateIndexRequest.extra_params:type_name -> milvus.proto.common.KeyValuePair + 58, // 12: milvus.proto.internal.SubSearchRequest.dsl_type:type_name -> milvus.proto.common.DslType + 57, // 13: milvus.proto.internal.SearchRequest.base:type_name -> milvus.proto.common.MsgBase + 58, // 14: milvus.proto.internal.SearchRequest.dsl_type:type_name -> milvus.proto.common.DslType 15, // 15: milvus.proto.internal.SearchRequest.sub_reqs:type_name -> milvus.proto.internal.SubSearchRequest - 57, // 16: milvus.proto.internal.SearchRequest.consistency_level:type_name -> milvus.proto.common.ConsistencyLevel - 55, // 17: milvus.proto.internal.SearchResults.base:type_name -> milvus.proto.common.MsgBase - 54, // 18: milvus.proto.internal.SearchResults.status:type_name -> milvus.proto.common.Status + 59, // 16: milvus.proto.internal.SearchRequest.consistency_level:type_name -> milvus.proto.common.ConsistencyLevel + 57, // 17: milvus.proto.internal.SearchResults.base:type_name -> milvus.proto.common.MsgBase + 56, // 18: milvus.proto.internal.SearchResults.status:type_name -> milvus.proto.common.Status 19, // 19: milvus.proto.internal.SearchResults.costAggregation:type_name -> milvus.proto.internal.CostAggregation - 51, // 20: milvus.proto.internal.SearchResults.channels_mvcc:type_name -> milvus.proto.internal.SearchResults.ChannelsMvccEntry + 53, // 20: milvus.proto.internal.SearchResults.channels_mvcc:type_name -> milvus.proto.internal.SearchResults.ChannelsMvccEntry 17, // 21: milvus.proto.internal.SearchResults.sub_results:type_name -> milvus.proto.internal.SubSearchResults - 55, // 22: milvus.proto.internal.RetrieveRequest.base:type_name -> milvus.proto.common.MsgBase - 57, // 23: milvus.proto.internal.RetrieveRequest.consistency_level:type_name -> milvus.proto.common.ConsistencyLevel - 55, // 24: milvus.proto.internal.RetrieveResults.base:type_name -> milvus.proto.common.MsgBase - 54, // 25: milvus.proto.internal.RetrieveResults.status:type_name -> milvus.proto.common.Status - 58, // 26: milvus.proto.internal.RetrieveResults.ids:type_name -> milvus.proto.schema.IDs - 59, // 27: milvus.proto.internal.RetrieveResults.fields_data:type_name -> milvus.proto.schema.FieldData + 57, // 22: milvus.proto.internal.RetrieveRequest.base:type_name -> milvus.proto.common.MsgBase + 59, // 23: milvus.proto.internal.RetrieveRequest.consistency_level:type_name -> milvus.proto.common.ConsistencyLevel + 57, // 24: milvus.proto.internal.RetrieveResults.base:type_name -> milvus.proto.common.MsgBase + 56, // 25: milvus.proto.internal.RetrieveResults.status:type_name -> milvus.proto.common.Status + 60, // 26: milvus.proto.internal.RetrieveResults.ids:type_name -> milvus.proto.schema.IDs + 61, // 27: milvus.proto.internal.RetrieveResults.fields_data:type_name -> milvus.proto.schema.FieldData 19, // 28: milvus.proto.internal.RetrieveResults.costAggregation:type_name -> milvus.proto.internal.CostAggregation - 55, // 29: milvus.proto.internal.LoadIndex.base:type_name -> milvus.proto.common.MsgBase - 53, // 30: milvus.proto.internal.LoadIndex.index_params:type_name -> milvus.proto.common.KeyValuePair - 53, // 31: milvus.proto.internal.IndexStats.index_params:type_name -> milvus.proto.common.KeyValuePair + 57, // 29: milvus.proto.internal.LoadIndex.base:type_name -> milvus.proto.common.MsgBase + 55, // 30: milvus.proto.internal.LoadIndex.index_params:type_name -> milvus.proto.common.KeyValuePair + 55, // 31: milvus.proto.internal.IndexStats.index_params:type_name -> milvus.proto.common.KeyValuePair 23, // 32: milvus.proto.internal.FieldStats.index_stats:type_name -> milvus.proto.internal.IndexStats - 55, // 33: milvus.proto.internal.ChannelTimeTickMsg.base:type_name -> milvus.proto.common.MsgBase - 55, // 34: milvus.proto.internal.ListPolicyRequest.base:type_name -> milvus.proto.common.MsgBase - 54, // 35: milvus.proto.internal.ListPolicyResponse.status:type_name -> milvus.proto.common.Status - 60, // 36: milvus.proto.internal.ListPolicyResponse.privilege_groups:type_name -> milvus.proto.milvus.PrivilegeGroupInfo - 55, // 37: milvus.proto.internal.ShowConfigurationsRequest.base:type_name -> milvus.proto.common.MsgBase - 54, // 38: milvus.proto.internal.ShowConfigurationsResponse.status:type_name -> milvus.proto.common.Status - 53, // 39: milvus.proto.internal.ShowConfigurationsResponse.configuations:type_name -> milvus.proto.common.KeyValuePair + 57, // 33: milvus.proto.internal.ChannelTimeTickMsg.base:type_name -> milvus.proto.common.MsgBase + 57, // 34: milvus.proto.internal.ListPolicyRequest.base:type_name -> milvus.proto.common.MsgBase + 56, // 35: milvus.proto.internal.ListPolicyResponse.status:type_name -> milvus.proto.common.Status + 62, // 36: milvus.proto.internal.ListPolicyResponse.privilege_groups:type_name -> milvus.proto.milvus.PrivilegeGroupInfo + 57, // 37: milvus.proto.internal.ShowConfigurationsRequest.base:type_name -> milvus.proto.common.MsgBase + 56, // 38: milvus.proto.internal.ShowConfigurationsResponse.status:type_name -> milvus.proto.common.Status + 55, // 39: milvus.proto.internal.ShowConfigurationsResponse.configuations:type_name -> milvus.proto.common.KeyValuePair 1, // 40: milvus.proto.internal.Rate.rt:type_name -> milvus.proto.internal.RateType - 61, // 41: milvus.proto.internal.ImportRequestInternal.schema:type_name -> milvus.proto.schema.CollectionSchema + 63, // 41: milvus.proto.internal.ImportRequestInternal.schema:type_name -> milvus.proto.schema.CollectionSchema 33, // 42: milvus.proto.internal.ImportRequestInternal.files:type_name -> milvus.proto.internal.ImportFile - 53, // 43: milvus.proto.internal.ImportRequestInternal.options:type_name -> milvus.proto.common.KeyValuePair + 55, // 43: milvus.proto.internal.ImportRequestInternal.options:type_name -> milvus.proto.common.KeyValuePair 33, // 44: milvus.proto.internal.ImportRequest.files:type_name -> milvus.proto.internal.ImportFile - 53, // 45: milvus.proto.internal.ImportRequest.options:type_name -> milvus.proto.common.KeyValuePair - 54, // 46: milvus.proto.internal.ImportResponse.status:type_name -> milvus.proto.common.Status - 54, // 47: milvus.proto.internal.GetImportProgressResponse.status:type_name -> milvus.proto.common.Status + 55, // 45: milvus.proto.internal.ImportRequest.options:type_name -> milvus.proto.common.KeyValuePair + 56, // 46: milvus.proto.internal.ImportResponse.status:type_name -> milvus.proto.common.Status + 56, // 47: milvus.proto.internal.GetImportProgressResponse.status:type_name -> milvus.proto.common.Status 2, // 48: milvus.proto.internal.GetImportProgressResponse.state:type_name -> milvus.proto.internal.ImportJobState 38, // 49: milvus.proto.internal.GetImportProgressResponse.task_progresses:type_name -> milvus.proto.internal.ImportTaskProgress - 54, // 50: milvus.proto.internal.ListImportsResponse.status:type_name -> milvus.proto.common.Status + 56, // 50: milvus.proto.internal.ListImportsResponse.status:type_name -> milvus.proto.common.Status 2, // 51: milvus.proto.internal.ListImportsResponse.states:type_name -> milvus.proto.internal.ImportJobState - 62, // 52: milvus.proto.internal.SegmentInfo.state:type_name -> milvus.proto.common.SegmentState - 63, // 53: milvus.proto.internal.SegmentInfo.level:type_name -> milvus.proto.common.SegmentLevel + 64, // 52: milvus.proto.internal.SegmentInfo.state:type_name -> milvus.proto.common.SegmentState + 65, // 53: milvus.proto.internal.SegmentInfo.level:type_name -> milvus.proto.common.SegmentLevel 44, // 54: milvus.proto.internal.SegmentInfo.insert_logs:type_name -> milvus.proto.internal.FieldBinlog 44, // 55: milvus.proto.internal.SegmentInfo.delta_logs:type_name -> milvus.proto.internal.FieldBinlog 44, // 56: milvus.proto.internal.SegmentInfo.stats_logs:type_name -> milvus.proto.internal.FieldBinlog - 54, // 57: milvus.proto.internal.GetSegmentsInfoResponse.status:type_name -> milvus.proto.common.Status + 56, // 57: milvus.proto.internal.GetSegmentsInfoResponse.status:type_name -> milvus.proto.common.Status 45, // 58: milvus.proto.internal.GetSegmentsInfoResponse.segmentInfos:type_name -> milvus.proto.internal.SegmentInfo - 55, // 59: milvus.proto.internal.GetQuotaMetricsRequest.base:type_name -> milvus.proto.common.MsgBase - 54, // 60: milvus.proto.internal.GetQuotaMetricsResponse.status:type_name -> milvus.proto.common.Status + 57, // 59: milvus.proto.internal.GetQuotaMetricsRequest.base:type_name -> milvus.proto.common.MsgBase + 56, // 60: milvus.proto.internal.GetQuotaMetricsResponse.status:type_name -> milvus.proto.common.Status 49, // 61: milvus.proto.internal.SyncFileResourceRequest.resources:type_name -> milvus.proto.internal.FileResourceInfo - 62, // [62:62] is the sub-list for method output_type - 62, // [62:62] is the sub-list for method input_type - 62, // [62:62] is the sub-list for extension type_name - 62, // [62:62] is the sub-list for extension extendee - 0, // [0:62] is the sub-list for field type_name + 57, // 62: milvus.proto.internal.BackupEzkRequest.base:type_name -> milvus.proto.common.MsgBase + 56, // 63: milvus.proto.internal.BackupEzkResponse.status:type_name -> milvus.proto.common.Status + 64, // [64:64] is the sub-list for method output_type + 64, // [64:64] is the sub-list for method input_type + 64, // [64:64] is the sub-list for extension type_name + 64, // [64:64] is the sub-list for extension extendee + 0, // [0:64] is the sub-list for field type_name } func init() { file_internal_proto_init() } @@ -5641,6 +5767,30 @@ func file_internal_proto_init() { return nil } } + file_internal_proto_msgTypes[48].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*BackupEzkRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_internal_proto_msgTypes[49].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*BackupEzkResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } type x struct{} out := protoimpl.TypeBuilder{ @@ -5648,7 +5798,7 @@ func file_internal_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_internal_proto_rawDesc, NumEnums: 3, - NumMessages: 49, + NumMessages: 51, NumExtensions: 0, NumServices: 0, }, diff --git a/pkg/proto/root_coord.proto b/pkg/proto/root_coord.proto index d00d2e4d3b..7025018b91 100644 --- a/pkg/proto/root_coord.proto +++ b/pkg/proto/root_coord.proto @@ -169,6 +169,7 @@ service RootCoord { rpc AlterDatabase(AlterDatabaseRequest) returns(common.Status){} rpc GetQuotaMetrics(internal.GetQuotaMetricsRequest) returns (internal.GetQuotaMetricsResponse) {} + rpc BackupEzk(internal.BackupEzkRequest) returns (internal.BackupEzkResponse) {} } message AllocTimestampRequest { diff --git a/pkg/proto/rootcoordpb/root_coord.pb.go b/pkg/proto/rootcoordpb/root_coord.pb.go index b9023e33e4..f265827f84 100644 --- a/pkg/proto/rootcoordpb/root_coord.pb.go +++ b/pkg/proto/rootcoordpb/root_coord.pb.go @@ -1466,7 +1466,7 @@ var file_root_coord_proto_rawDesc = []byte{ 0x03, 0x28, 0x0b, 0x32, 0x25, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x72, 0x6f, 0x6f, 0x74, 0x63, 0x6f, 0x6f, 0x72, 0x64, 0x2e, 0x44, 0x42, 0x43, 0x6f, 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x52, 0x0d, 0x64, 0x62, 0x43, 0x6f, - 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x32, 0x81, 0x32, 0x0a, 0x09, 0x52, 0x6f, + 0x6c, 0x6c, 0x65, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x32, 0xe3, 0x32, 0x0a, 0x09, 0x52, 0x6f, 0x6f, 0x74, 0x43, 0x6f, 0x6f, 0x72, 0x64, 0x12, 0x6c, 0x0a, 0x12, 0x47, 0x65, 0x74, 0x43, 0x6f, 0x6d, 0x70, 0x6f, 0x6e, 0x65, 0x6e, 0x74, 0x53, 0x74, 0x61, 0x74, 0x65, 0x73, 0x12, 0x2e, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x6d, 0x69, 0x6c, @@ -1866,11 +1866,17 @@ var file_root_coord_proto_rawDesc = []byte{ 0x72, 0x69, 0x63, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x1a, 0x2e, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x47, 0x65, 0x74, 0x51, 0x75, 0x6f, 0x74, 0x61, 0x4d, 0x65, 0x74, 0x72, - 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, 0x36, 0x5a, - 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x69, 0x6c, 0x76, - 0x75, 0x73, 0x2d, 0x69, 0x6f, 0x2f, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2f, 0x70, 0x6b, 0x67, - 0x2f, 0x76, 0x32, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x6f, 0x6f, 0x74, 0x63, 0x6f, - 0x6f, 0x72, 0x64, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x69, 0x63, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x12, 0x60, 0x0a, + 0x09, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x45, 0x7a, 0x6b, 0x12, 0x27, 0x2e, 0x6d, 0x69, 0x6c, + 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, + 0x61, 0x6c, 0x2e, 0x42, 0x61, 0x63, 0x6b, 0x75, 0x70, 0x45, 0x7a, 0x6b, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x1a, 0x28, 0x2e, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2e, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x2e, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2e, 0x42, 0x61, 0x63, 0x6b, + 0x75, 0x70, 0x45, 0x7a, 0x6b, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x00, 0x42, + 0x36, 0x5a, 0x34, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, 0x6d, 0x69, + 0x6c, 0x76, 0x75, 0x73, 0x2d, 0x69, 0x6f, 0x2f, 0x6d, 0x69, 0x6c, 0x76, 0x75, 0x73, 0x2f, 0x70, + 0x6b, 0x67, 0x2f, 0x76, 0x32, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x2f, 0x72, 0x6f, 0x6f, 0x74, + 0x63, 0x6f, 0x6f, 0x72, 0x64, 0x70, 0x62, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1967,28 +1973,30 @@ var file_root_coord_proto_goTypes = []interface{}{ (*milvuspb.DropDatabaseRequest)(nil), // 77: milvus.proto.milvus.DropDatabaseRequest (*milvuspb.ListDatabasesRequest)(nil), // 78: milvus.proto.milvus.ListDatabasesRequest (*internalpb.GetQuotaMetricsRequest)(nil), // 79: milvus.proto.internal.GetQuotaMetricsRequest - (*milvuspb.ComponentStates)(nil), // 80: milvus.proto.milvus.ComponentStates - (*milvuspb.StringResponse)(nil), // 81: milvus.proto.milvus.StringResponse - (*milvuspb.TruncateCollectionResponse)(nil), // 82: milvus.proto.milvus.TruncateCollectionResponse - (*milvuspb.BoolResponse)(nil), // 83: milvus.proto.milvus.BoolResponse - (*milvuspb.DescribeCollectionResponse)(nil), // 84: milvus.proto.milvus.DescribeCollectionResponse - (*milvuspb.DescribeAliasResponse)(nil), // 85: milvus.proto.milvus.DescribeAliasResponse - (*milvuspb.ListAliasesResponse)(nil), // 86: milvus.proto.milvus.ListAliasesResponse - (*milvuspb.ShowCollectionsResponse)(nil), // 87: milvus.proto.milvus.ShowCollectionsResponse - (*milvuspb.ShowPartitionsResponse)(nil), // 88: milvus.proto.milvus.ShowPartitionsResponse - (*milvuspb.ShowSegmentsResponse)(nil), // 89: milvus.proto.milvus.ShowSegmentsResponse - (*internalpb.ShowConfigurationsResponse)(nil), // 90: milvus.proto.internal.ShowConfigurationsResponse - (*milvuspb.GetMetricsResponse)(nil), // 91: milvus.proto.milvus.GetMetricsResponse - (*milvuspb.ListCredUsersResponse)(nil), // 92: milvus.proto.milvus.ListCredUsersResponse - (*milvuspb.SelectRoleResponse)(nil), // 93: milvus.proto.milvus.SelectRoleResponse - (*milvuspb.SelectUserResponse)(nil), // 94: milvus.proto.milvus.SelectUserResponse - (*milvuspb.SelectGrantResponse)(nil), // 95: milvus.proto.milvus.SelectGrantResponse - (*internalpb.ListPolicyResponse)(nil), // 96: milvus.proto.internal.ListPolicyResponse - (*milvuspb.BackupRBACMetaResponse)(nil), // 97: milvus.proto.milvus.BackupRBACMetaResponse - (*milvuspb.ListPrivilegeGroupsResponse)(nil), // 98: milvus.proto.milvus.ListPrivilegeGroupsResponse - (*milvuspb.CheckHealthResponse)(nil), // 99: milvus.proto.milvus.CheckHealthResponse - (*milvuspb.ListDatabasesResponse)(nil), // 100: milvus.proto.milvus.ListDatabasesResponse - (*internalpb.GetQuotaMetricsResponse)(nil), // 101: milvus.proto.internal.GetQuotaMetricsResponse + (*internalpb.BackupEzkRequest)(nil), // 80: milvus.proto.internal.BackupEzkRequest + (*milvuspb.ComponentStates)(nil), // 81: milvus.proto.milvus.ComponentStates + (*milvuspb.StringResponse)(nil), // 82: milvus.proto.milvus.StringResponse + (*milvuspb.TruncateCollectionResponse)(nil), // 83: milvus.proto.milvus.TruncateCollectionResponse + (*milvuspb.BoolResponse)(nil), // 84: milvus.proto.milvus.BoolResponse + (*milvuspb.DescribeCollectionResponse)(nil), // 85: milvus.proto.milvus.DescribeCollectionResponse + (*milvuspb.DescribeAliasResponse)(nil), // 86: milvus.proto.milvus.DescribeAliasResponse + (*milvuspb.ListAliasesResponse)(nil), // 87: milvus.proto.milvus.ListAliasesResponse + (*milvuspb.ShowCollectionsResponse)(nil), // 88: milvus.proto.milvus.ShowCollectionsResponse + (*milvuspb.ShowPartitionsResponse)(nil), // 89: milvus.proto.milvus.ShowPartitionsResponse + (*milvuspb.ShowSegmentsResponse)(nil), // 90: milvus.proto.milvus.ShowSegmentsResponse + (*internalpb.ShowConfigurationsResponse)(nil), // 91: milvus.proto.internal.ShowConfigurationsResponse + (*milvuspb.GetMetricsResponse)(nil), // 92: milvus.proto.milvus.GetMetricsResponse + (*milvuspb.ListCredUsersResponse)(nil), // 93: milvus.proto.milvus.ListCredUsersResponse + (*milvuspb.SelectRoleResponse)(nil), // 94: milvus.proto.milvus.SelectRoleResponse + (*milvuspb.SelectUserResponse)(nil), // 95: milvus.proto.milvus.SelectUserResponse + (*milvuspb.SelectGrantResponse)(nil), // 96: milvus.proto.milvus.SelectGrantResponse + (*internalpb.ListPolicyResponse)(nil), // 97: milvus.proto.internal.ListPolicyResponse + (*milvuspb.BackupRBACMetaResponse)(nil), // 98: milvus.proto.milvus.BackupRBACMetaResponse + (*milvuspb.ListPrivilegeGroupsResponse)(nil), // 99: milvus.proto.milvus.ListPrivilegeGroupsResponse + (*milvuspb.CheckHealthResponse)(nil), // 100: milvus.proto.milvus.CheckHealthResponse + (*milvuspb.ListDatabasesResponse)(nil), // 101: milvus.proto.milvus.ListDatabasesResponse + (*internalpb.GetQuotaMetricsResponse)(nil), // 102: milvus.proto.internal.GetQuotaMetricsResponse + (*internalpb.BackupEzkResponse)(nil), // 103: milvus.proto.internal.BackupEzkResponse } var file_root_coord_proto_depIdxs = []int32{ 22, // 0: milvus.proto.rootcoord.AllocTimestampRequest.base:type_name -> milvus.proto.common.MsgBase @@ -2080,70 +2088,72 @@ var file_root_coord_proto_depIdxs = []int32{ 10, // 86: milvus.proto.rootcoord.RootCoord.DescribeDatabase:input_type -> milvus.proto.rootcoord.DescribeDatabaseRequest 12, // 87: milvus.proto.rootcoord.RootCoord.AlterDatabase:input_type -> milvus.proto.rootcoord.AlterDatabaseRequest 79, // 88: milvus.proto.rootcoord.RootCoord.GetQuotaMetrics:input_type -> milvus.proto.internal.GetQuotaMetricsRequest - 80, // 89: milvus.proto.rootcoord.RootCoord.GetComponentStates:output_type -> milvus.proto.milvus.ComponentStates - 81, // 90: milvus.proto.rootcoord.RootCoord.GetTimeTickChannel:output_type -> milvus.proto.milvus.StringResponse - 81, // 91: milvus.proto.rootcoord.RootCoord.GetStatisticsChannel:output_type -> milvus.proto.milvus.StringResponse - 23, // 92: milvus.proto.rootcoord.RootCoord.CreateCollection:output_type -> milvus.proto.common.Status - 23, // 93: milvus.proto.rootcoord.RootCoord.DropCollection:output_type -> milvus.proto.common.Status - 82, // 94: milvus.proto.rootcoord.RootCoord.TruncateCollection:output_type -> milvus.proto.milvus.TruncateCollectionResponse - 23, // 95: milvus.proto.rootcoord.RootCoord.AddCollectionField:output_type -> milvus.proto.common.Status - 83, // 96: milvus.proto.rootcoord.RootCoord.HasCollection:output_type -> milvus.proto.milvus.BoolResponse - 84, // 97: milvus.proto.rootcoord.RootCoord.DescribeCollection:output_type -> milvus.proto.milvus.DescribeCollectionResponse - 84, // 98: milvus.proto.rootcoord.RootCoord.DescribeCollectionInternal:output_type -> milvus.proto.milvus.DescribeCollectionResponse - 23, // 99: milvus.proto.rootcoord.RootCoord.CreateAlias:output_type -> milvus.proto.common.Status - 23, // 100: milvus.proto.rootcoord.RootCoord.DropAlias:output_type -> milvus.proto.common.Status - 23, // 101: milvus.proto.rootcoord.RootCoord.AlterAlias:output_type -> milvus.proto.common.Status - 85, // 102: milvus.proto.rootcoord.RootCoord.DescribeAlias:output_type -> milvus.proto.milvus.DescribeAliasResponse - 86, // 103: milvus.proto.rootcoord.RootCoord.ListAliases:output_type -> milvus.proto.milvus.ListAliasesResponse - 87, // 104: milvus.proto.rootcoord.RootCoord.ShowCollections:output_type -> milvus.proto.milvus.ShowCollectionsResponse - 19, // 105: milvus.proto.rootcoord.RootCoord.ShowCollectionIDs:output_type -> milvus.proto.rootcoord.ShowCollectionIDsResponse - 23, // 106: milvus.proto.rootcoord.RootCoord.AlterCollection:output_type -> milvus.proto.common.Status - 23, // 107: milvus.proto.rootcoord.RootCoord.AlterCollectionField:output_type -> milvus.proto.common.Status - 23, // 108: milvus.proto.rootcoord.RootCoord.AddCollectionFunction:output_type -> milvus.proto.common.Status - 23, // 109: milvus.proto.rootcoord.RootCoord.AlterCollectionFunction:output_type -> milvus.proto.common.Status - 23, // 110: milvus.proto.rootcoord.RootCoord.DropCollectionFunction:output_type -> milvus.proto.common.Status - 23, // 111: milvus.proto.rootcoord.RootCoord.CreatePartition:output_type -> milvus.proto.common.Status - 23, // 112: milvus.proto.rootcoord.RootCoord.DropPartition:output_type -> milvus.proto.common.Status - 83, // 113: milvus.proto.rootcoord.RootCoord.HasPartition:output_type -> milvus.proto.milvus.BoolResponse - 88, // 114: milvus.proto.rootcoord.RootCoord.ShowPartitions:output_type -> milvus.proto.milvus.ShowPartitionsResponse - 88, // 115: milvus.proto.rootcoord.RootCoord.ShowPartitionsInternal:output_type -> milvus.proto.milvus.ShowPartitionsResponse - 89, // 116: milvus.proto.rootcoord.RootCoord.ShowSegments:output_type -> milvus.proto.milvus.ShowSegmentsResponse - 14, // 117: milvus.proto.rootcoord.RootCoord.GetPChannelInfo:output_type -> milvus.proto.rootcoord.GetPChannelInfoResponse - 1, // 118: milvus.proto.rootcoord.RootCoord.AllocTimestamp:output_type -> milvus.proto.rootcoord.AllocTimestampResponse - 3, // 119: milvus.proto.rootcoord.RootCoord.AllocID:output_type -> milvus.proto.rootcoord.AllocIDResponse - 23, // 120: milvus.proto.rootcoord.RootCoord.UpdateChannelTimeTick:output_type -> milvus.proto.common.Status - 23, // 121: milvus.proto.rootcoord.RootCoord.InvalidateCollectionMetaCache:output_type -> milvus.proto.common.Status - 90, // 122: milvus.proto.rootcoord.RootCoord.ShowConfigurations:output_type -> milvus.proto.internal.ShowConfigurationsResponse - 91, // 123: milvus.proto.rootcoord.RootCoord.GetMetrics:output_type -> milvus.proto.milvus.GetMetricsResponse - 23, // 124: milvus.proto.rootcoord.RootCoord.CreateCredential:output_type -> milvus.proto.common.Status - 23, // 125: milvus.proto.rootcoord.RootCoord.UpdateCredential:output_type -> milvus.proto.common.Status - 23, // 126: milvus.proto.rootcoord.RootCoord.DeleteCredential:output_type -> milvus.proto.common.Status - 92, // 127: milvus.proto.rootcoord.RootCoord.ListCredUsers:output_type -> milvus.proto.milvus.ListCredUsersResponse - 9, // 128: milvus.proto.rootcoord.RootCoord.GetCredential:output_type -> milvus.proto.rootcoord.GetCredentialResponse - 23, // 129: milvus.proto.rootcoord.RootCoord.CreateRole:output_type -> milvus.proto.common.Status - 23, // 130: milvus.proto.rootcoord.RootCoord.DropRole:output_type -> milvus.proto.common.Status - 23, // 131: milvus.proto.rootcoord.RootCoord.OperateUserRole:output_type -> milvus.proto.common.Status - 93, // 132: milvus.proto.rootcoord.RootCoord.SelectRole:output_type -> milvus.proto.milvus.SelectRoleResponse - 94, // 133: milvus.proto.rootcoord.RootCoord.SelectUser:output_type -> milvus.proto.milvus.SelectUserResponse - 23, // 134: milvus.proto.rootcoord.RootCoord.OperatePrivilege:output_type -> milvus.proto.common.Status - 95, // 135: milvus.proto.rootcoord.RootCoord.SelectGrant:output_type -> milvus.proto.milvus.SelectGrantResponse - 96, // 136: milvus.proto.rootcoord.RootCoord.ListPolicy:output_type -> milvus.proto.internal.ListPolicyResponse - 97, // 137: milvus.proto.rootcoord.RootCoord.BackupRBAC:output_type -> milvus.proto.milvus.BackupRBACMetaResponse - 23, // 138: milvus.proto.rootcoord.RootCoord.RestoreRBAC:output_type -> milvus.proto.common.Status - 23, // 139: milvus.proto.rootcoord.RootCoord.CreatePrivilegeGroup:output_type -> milvus.proto.common.Status - 23, // 140: milvus.proto.rootcoord.RootCoord.DropPrivilegeGroup:output_type -> milvus.proto.common.Status - 98, // 141: milvus.proto.rootcoord.RootCoord.ListPrivilegeGroups:output_type -> milvus.proto.milvus.ListPrivilegeGroupsResponse - 23, // 142: milvus.proto.rootcoord.RootCoord.OperatePrivilegeGroup:output_type -> milvus.proto.common.Status - 99, // 143: milvus.proto.rootcoord.RootCoord.CheckHealth:output_type -> milvus.proto.milvus.CheckHealthResponse - 23, // 144: milvus.proto.rootcoord.RootCoord.RenameCollection:output_type -> milvus.proto.common.Status - 23, // 145: milvus.proto.rootcoord.RootCoord.CreateDatabase:output_type -> milvus.proto.common.Status - 23, // 146: milvus.proto.rootcoord.RootCoord.DropDatabase:output_type -> milvus.proto.common.Status - 100, // 147: milvus.proto.rootcoord.RootCoord.ListDatabases:output_type -> milvus.proto.milvus.ListDatabasesResponse - 11, // 148: milvus.proto.rootcoord.RootCoord.DescribeDatabase:output_type -> milvus.proto.rootcoord.DescribeDatabaseResponse - 23, // 149: milvus.proto.rootcoord.RootCoord.AlterDatabase:output_type -> milvus.proto.common.Status - 101, // 150: milvus.proto.rootcoord.RootCoord.GetQuotaMetrics:output_type -> milvus.proto.internal.GetQuotaMetricsResponse - 89, // [89:151] is the sub-list for method output_type - 27, // [27:89] is the sub-list for method input_type + 80, // 89: milvus.proto.rootcoord.RootCoord.BackupEzk:input_type -> milvus.proto.internal.BackupEzkRequest + 81, // 90: milvus.proto.rootcoord.RootCoord.GetComponentStates:output_type -> milvus.proto.milvus.ComponentStates + 82, // 91: milvus.proto.rootcoord.RootCoord.GetTimeTickChannel:output_type -> milvus.proto.milvus.StringResponse + 82, // 92: milvus.proto.rootcoord.RootCoord.GetStatisticsChannel:output_type -> milvus.proto.milvus.StringResponse + 23, // 93: milvus.proto.rootcoord.RootCoord.CreateCollection:output_type -> milvus.proto.common.Status + 23, // 94: milvus.proto.rootcoord.RootCoord.DropCollection:output_type -> milvus.proto.common.Status + 83, // 95: milvus.proto.rootcoord.RootCoord.TruncateCollection:output_type -> milvus.proto.milvus.TruncateCollectionResponse + 23, // 96: milvus.proto.rootcoord.RootCoord.AddCollectionField:output_type -> milvus.proto.common.Status + 84, // 97: milvus.proto.rootcoord.RootCoord.HasCollection:output_type -> milvus.proto.milvus.BoolResponse + 85, // 98: milvus.proto.rootcoord.RootCoord.DescribeCollection:output_type -> milvus.proto.milvus.DescribeCollectionResponse + 85, // 99: milvus.proto.rootcoord.RootCoord.DescribeCollectionInternal:output_type -> milvus.proto.milvus.DescribeCollectionResponse + 23, // 100: milvus.proto.rootcoord.RootCoord.CreateAlias:output_type -> milvus.proto.common.Status + 23, // 101: milvus.proto.rootcoord.RootCoord.DropAlias:output_type -> milvus.proto.common.Status + 23, // 102: milvus.proto.rootcoord.RootCoord.AlterAlias:output_type -> milvus.proto.common.Status + 86, // 103: milvus.proto.rootcoord.RootCoord.DescribeAlias:output_type -> milvus.proto.milvus.DescribeAliasResponse + 87, // 104: milvus.proto.rootcoord.RootCoord.ListAliases:output_type -> milvus.proto.milvus.ListAliasesResponse + 88, // 105: milvus.proto.rootcoord.RootCoord.ShowCollections:output_type -> milvus.proto.milvus.ShowCollectionsResponse + 19, // 106: milvus.proto.rootcoord.RootCoord.ShowCollectionIDs:output_type -> milvus.proto.rootcoord.ShowCollectionIDsResponse + 23, // 107: milvus.proto.rootcoord.RootCoord.AlterCollection:output_type -> milvus.proto.common.Status + 23, // 108: milvus.proto.rootcoord.RootCoord.AlterCollectionField:output_type -> milvus.proto.common.Status + 23, // 109: milvus.proto.rootcoord.RootCoord.AddCollectionFunction:output_type -> milvus.proto.common.Status + 23, // 110: milvus.proto.rootcoord.RootCoord.AlterCollectionFunction:output_type -> milvus.proto.common.Status + 23, // 111: milvus.proto.rootcoord.RootCoord.DropCollectionFunction:output_type -> milvus.proto.common.Status + 23, // 112: milvus.proto.rootcoord.RootCoord.CreatePartition:output_type -> milvus.proto.common.Status + 23, // 113: milvus.proto.rootcoord.RootCoord.DropPartition:output_type -> milvus.proto.common.Status + 84, // 114: milvus.proto.rootcoord.RootCoord.HasPartition:output_type -> milvus.proto.milvus.BoolResponse + 89, // 115: milvus.proto.rootcoord.RootCoord.ShowPartitions:output_type -> milvus.proto.milvus.ShowPartitionsResponse + 89, // 116: milvus.proto.rootcoord.RootCoord.ShowPartitionsInternal:output_type -> milvus.proto.milvus.ShowPartitionsResponse + 90, // 117: milvus.proto.rootcoord.RootCoord.ShowSegments:output_type -> milvus.proto.milvus.ShowSegmentsResponse + 14, // 118: milvus.proto.rootcoord.RootCoord.GetPChannelInfo:output_type -> milvus.proto.rootcoord.GetPChannelInfoResponse + 1, // 119: milvus.proto.rootcoord.RootCoord.AllocTimestamp:output_type -> milvus.proto.rootcoord.AllocTimestampResponse + 3, // 120: milvus.proto.rootcoord.RootCoord.AllocID:output_type -> milvus.proto.rootcoord.AllocIDResponse + 23, // 121: milvus.proto.rootcoord.RootCoord.UpdateChannelTimeTick:output_type -> milvus.proto.common.Status + 23, // 122: milvus.proto.rootcoord.RootCoord.InvalidateCollectionMetaCache:output_type -> milvus.proto.common.Status + 91, // 123: milvus.proto.rootcoord.RootCoord.ShowConfigurations:output_type -> milvus.proto.internal.ShowConfigurationsResponse + 92, // 124: milvus.proto.rootcoord.RootCoord.GetMetrics:output_type -> milvus.proto.milvus.GetMetricsResponse + 23, // 125: milvus.proto.rootcoord.RootCoord.CreateCredential:output_type -> milvus.proto.common.Status + 23, // 126: milvus.proto.rootcoord.RootCoord.UpdateCredential:output_type -> milvus.proto.common.Status + 23, // 127: milvus.proto.rootcoord.RootCoord.DeleteCredential:output_type -> milvus.proto.common.Status + 93, // 128: milvus.proto.rootcoord.RootCoord.ListCredUsers:output_type -> milvus.proto.milvus.ListCredUsersResponse + 9, // 129: milvus.proto.rootcoord.RootCoord.GetCredential:output_type -> milvus.proto.rootcoord.GetCredentialResponse + 23, // 130: milvus.proto.rootcoord.RootCoord.CreateRole:output_type -> milvus.proto.common.Status + 23, // 131: milvus.proto.rootcoord.RootCoord.DropRole:output_type -> milvus.proto.common.Status + 23, // 132: milvus.proto.rootcoord.RootCoord.OperateUserRole:output_type -> milvus.proto.common.Status + 94, // 133: milvus.proto.rootcoord.RootCoord.SelectRole:output_type -> milvus.proto.milvus.SelectRoleResponse + 95, // 134: milvus.proto.rootcoord.RootCoord.SelectUser:output_type -> milvus.proto.milvus.SelectUserResponse + 23, // 135: milvus.proto.rootcoord.RootCoord.OperatePrivilege:output_type -> milvus.proto.common.Status + 96, // 136: milvus.proto.rootcoord.RootCoord.SelectGrant:output_type -> milvus.proto.milvus.SelectGrantResponse + 97, // 137: milvus.proto.rootcoord.RootCoord.ListPolicy:output_type -> milvus.proto.internal.ListPolicyResponse + 98, // 138: milvus.proto.rootcoord.RootCoord.BackupRBAC:output_type -> milvus.proto.milvus.BackupRBACMetaResponse + 23, // 139: milvus.proto.rootcoord.RootCoord.RestoreRBAC:output_type -> milvus.proto.common.Status + 23, // 140: milvus.proto.rootcoord.RootCoord.CreatePrivilegeGroup:output_type -> milvus.proto.common.Status + 23, // 141: milvus.proto.rootcoord.RootCoord.DropPrivilegeGroup:output_type -> milvus.proto.common.Status + 99, // 142: milvus.proto.rootcoord.RootCoord.ListPrivilegeGroups:output_type -> milvus.proto.milvus.ListPrivilegeGroupsResponse + 23, // 143: milvus.proto.rootcoord.RootCoord.OperatePrivilegeGroup:output_type -> milvus.proto.common.Status + 100, // 144: milvus.proto.rootcoord.RootCoord.CheckHealth:output_type -> milvus.proto.milvus.CheckHealthResponse + 23, // 145: milvus.proto.rootcoord.RootCoord.RenameCollection:output_type -> milvus.proto.common.Status + 23, // 146: milvus.proto.rootcoord.RootCoord.CreateDatabase:output_type -> milvus.proto.common.Status + 23, // 147: milvus.proto.rootcoord.RootCoord.DropDatabase:output_type -> milvus.proto.common.Status + 101, // 148: milvus.proto.rootcoord.RootCoord.ListDatabases:output_type -> milvus.proto.milvus.ListDatabasesResponse + 11, // 149: milvus.proto.rootcoord.RootCoord.DescribeDatabase:output_type -> milvus.proto.rootcoord.DescribeDatabaseResponse + 23, // 150: milvus.proto.rootcoord.RootCoord.AlterDatabase:output_type -> milvus.proto.common.Status + 102, // 151: milvus.proto.rootcoord.RootCoord.GetQuotaMetrics:output_type -> milvus.proto.internal.GetQuotaMetricsResponse + 103, // 152: milvus.proto.rootcoord.RootCoord.BackupEzk:output_type -> milvus.proto.internal.BackupEzkResponse + 90, // [90:153] is the sub-list for method output_type + 27, // [27:90] is the sub-list for method input_type 27, // [27:27] is the sub-list for extension type_name 27, // [27:27] is the sub-list for extension extendee 0, // [0:27] is the sub-list for field type_name diff --git a/pkg/proto/rootcoordpb/root_coord_grpc.pb.go b/pkg/proto/rootcoordpb/root_coord_grpc.pb.go index 9a2219daf8..317716c7d2 100644 --- a/pkg/proto/rootcoordpb/root_coord_grpc.pb.go +++ b/pkg/proto/rootcoordpb/root_coord_grpc.pb.go @@ -85,6 +85,7 @@ const ( RootCoord_DescribeDatabase_FullMethodName = "/milvus.proto.rootcoord.RootCoord/DescribeDatabase" RootCoord_AlterDatabase_FullMethodName = "/milvus.proto.rootcoord.RootCoord/AlterDatabase" RootCoord_GetQuotaMetrics_FullMethodName = "/milvus.proto.rootcoord.RootCoord/GetQuotaMetrics" + RootCoord_BackupEzk_FullMethodName = "/milvus.proto.rootcoord.RootCoord/BackupEzk" ) // RootCoordClient is the client API for RootCoord service. @@ -215,6 +216,7 @@ type RootCoordClient interface { DescribeDatabase(ctx context.Context, in *DescribeDatabaseRequest, opts ...grpc.CallOption) (*DescribeDatabaseResponse, error) AlterDatabase(ctx context.Context, in *AlterDatabaseRequest, opts ...grpc.CallOption) (*commonpb.Status, error) GetQuotaMetrics(ctx context.Context, in *internalpb.GetQuotaMetricsRequest, opts ...grpc.CallOption) (*internalpb.GetQuotaMetricsResponse, error) + BackupEzk(ctx context.Context, in *internalpb.BackupEzkRequest, opts ...grpc.CallOption) (*internalpb.BackupEzkResponse, error) } type rootCoordClient struct { @@ -783,6 +785,15 @@ func (c *rootCoordClient) GetQuotaMetrics(ctx context.Context, in *internalpb.Ge return out, nil } +func (c *rootCoordClient) BackupEzk(ctx context.Context, in *internalpb.BackupEzkRequest, opts ...grpc.CallOption) (*internalpb.BackupEzkResponse, error) { + out := new(internalpb.BackupEzkResponse) + err := c.cc.Invoke(ctx, RootCoord_BackupEzk_FullMethodName, in, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // RootCoordServer is the server API for RootCoord service. // All implementations should embed UnimplementedRootCoordServer // for forward compatibility @@ -911,6 +922,7 @@ type RootCoordServer interface { DescribeDatabase(context.Context, *DescribeDatabaseRequest) (*DescribeDatabaseResponse, error) AlterDatabase(context.Context, *AlterDatabaseRequest) (*commonpb.Status, error) GetQuotaMetrics(context.Context, *internalpb.GetQuotaMetricsRequest) (*internalpb.GetQuotaMetricsResponse, error) + BackupEzk(context.Context, *internalpb.BackupEzkRequest) (*internalpb.BackupEzkResponse, error) } // UnimplementedRootCoordServer should be embedded to have forward compatible implementations. @@ -1103,6 +1115,9 @@ func (UnimplementedRootCoordServer) AlterDatabase(context.Context, *AlterDatabas func (UnimplementedRootCoordServer) GetQuotaMetrics(context.Context, *internalpb.GetQuotaMetricsRequest) (*internalpb.GetQuotaMetricsResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method GetQuotaMetrics not implemented") } +func (UnimplementedRootCoordServer) BackupEzk(context.Context, *internalpb.BackupEzkRequest) (*internalpb.BackupEzkResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method BackupEzk not implemented") +} // UnsafeRootCoordServer may be embedded to opt out of forward compatibility for this service. // Use of this interface is not recommended, as added methods to RootCoordServer will @@ -2231,6 +2246,24 @@ func _RootCoord_GetQuotaMetrics_Handler(srv interface{}, ctx context.Context, de return interceptor(ctx, in, info, handler) } +func _RootCoord_BackupEzk_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(internalpb.BackupEzkRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(RootCoordServer).BackupEzk(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: RootCoord_BackupEzk_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(RootCoordServer).BackupEzk(ctx, req.(*internalpb.BackupEzkRequest)) + } + return interceptor(ctx, in, info, handler) +} + // RootCoord_ServiceDesc is the grpc.ServiceDesc for RootCoord service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -2486,6 +2519,10 @@ var RootCoord_ServiceDesc = grpc.ServiceDesc{ MethodName: "GetQuotaMetrics", Handler: _RootCoord_GetQuotaMetrics_Handler, }, + { + MethodName: "BackupEzk", + Handler: _RootCoord_BackupEzk_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "root_coord.proto",