diff --git a/.gitignore b/.gitignore index b6adfcbdb4..724c2ce2ac 100644 --- a/.gitignore +++ b/.gitignore @@ -103,4 +103,5 @@ cwrapper_rocksdb_build/ internal/proto/**/*.pb.go internal/core/src/pb/*.pb.h internal/core/src/pb/*.pb.cc +*.pb.go **/legacypb/*.pb.go diff --git a/internal/flushcommon/writebuffer/mock_manager.go b/internal/flushcommon/writebuffer/mock_manager.go index bfa6307be5..9c2e1490a0 100644 --- a/internal/flushcommon/writebuffer/mock_manager.go +++ b/internal/flushcommon/writebuffer/mock_manager.go @@ -46,11 +46,11 @@ type MockBufferManager_BufferData_Call struct { } // BufferData is a helper method to define mock.On call -// - channel string -// - insertMsgs []*msgstream.InsertMsg -// - deleteMsgs []*msgstream.DeleteMsg -// - startPos *msgpb.MsgPosition -// - endPos *msgpb.MsgPosition +// - channel string +// - insertMsgs []*msgstream.InsertMsg +// - deleteMsgs []*msgstream.DeleteMsg +// - startPos *msgpb.MsgPosition +// - endPos *msgpb.MsgPosition func (_e *MockBufferManager_Expecter) BufferData(channel interface{}, insertMsgs interface{}, deleteMsgs interface{}, startPos interface{}, endPos interface{}) *MockBufferManager_BufferData_Call { return &MockBufferManager_BufferData_Call{Call: _e.mock.On("BufferData", channel, insertMsgs, deleteMsgs, startPos, endPos)} } @@ -83,7 +83,7 @@ type MockBufferManager_DropChannel_Call struct { } // DropChannel is a helper method to define mock.On call -// - channel string +// - channel string func (_e *MockBufferManager_Expecter) DropChannel(channel interface{}) *MockBufferManager_DropChannel_Call { return &MockBufferManager_DropChannel_Call{Call: _e.mock.On("DropChannel", channel)} } @@ -116,8 +116,8 @@ type MockBufferManager_DropPartitions_Call struct { } // DropPartitions is a helper method to define mock.On call -// - channel string -// - partitionIDs []int64 +// - channel string +// - partitionIDs []int64 func (_e *MockBufferManager_Expecter) DropPartitions(channel interface{}, partitionIDs interface{}) *MockBufferManager_DropPartitions_Call { return &MockBufferManager_DropPartitions_Call{Call: _e.mock.On("DropPartitions", channel, partitionIDs)} } @@ -159,9 +159,9 @@ type MockBufferManager_FlushChannel_Call struct { } // FlushChannel is a helper method to define mock.On call -// - ctx context.Context -// - channel string -// - flushTs uint64 +// - ctx context.Context +// - channel string +// - flushTs uint64 func (_e *MockBufferManager_Expecter) FlushChannel(ctx interface{}, channel interface{}, flushTs interface{}) *MockBufferManager_FlushChannel_Call { return &MockBufferManager_FlushChannel_Call{Call: _e.mock.On("FlushChannel", ctx, channel, flushTs)} } @@ -222,7 +222,7 @@ type MockBufferManager_GetCheckpoint_Call struct { } // GetCheckpoint is a helper method to define mock.On call -// - channel string +// - channel string func (_e *MockBufferManager_Expecter) GetCheckpoint(channel interface{}) *MockBufferManager_GetCheckpoint_Call { return &MockBufferManager_GetCheckpoint_Call{Call: _e.mock.On("GetCheckpoint", channel)} } @@ -255,8 +255,8 @@ type MockBufferManager_NotifyCheckpointUpdated_Call struct { } // NotifyCheckpointUpdated is a helper method to define mock.On call -// - channel string -// - ts uint64 +// - channel string +// - ts uint64 func (_e *MockBufferManager_Expecter) NotifyCheckpointUpdated(channel interface{}, ts interface{}) *MockBufferManager_NotifyCheckpointUpdated_Call { return &MockBufferManager_NotifyCheckpointUpdated_Call{Call: _e.mock.On("NotifyCheckpointUpdated", channel, ts)} } @@ -305,10 +305,10 @@ type MockBufferManager_Register_Call struct { } // Register is a helper method to define mock.On call -// - channel string -// - _a1 metacache.MetaCache -// - storageV2Cache *metacache.StorageV2Cache -// - opts ...WriteBufferOption +// - channel string +// - _a1 metacache.MetaCache +// - storageV2Cache *metacache.StorageV2Cache +// - opts ...WriteBufferOption func (_e *MockBufferManager_Expecter) Register(channel interface{}, _a1 interface{}, storageV2Cache interface{}, opts ...interface{}) *MockBufferManager_Register_Call { return &MockBufferManager_Register_Call{Call: _e.mock.On("Register", append([]interface{}{channel, _a1, storageV2Cache}, opts...)...)} @@ -348,7 +348,7 @@ type MockBufferManager_RemoveChannel_Call struct { } // RemoveChannel is a helper method to define mock.On call -// - channel string +// - channel string func (_e *MockBufferManager_Expecter) RemoveChannel(channel interface{}) *MockBufferManager_RemoveChannel_Call { return &MockBufferManager_RemoveChannel_Call{Call: _e.mock.On("RemoveChannel", channel)} } @@ -390,9 +390,9 @@ type MockBufferManager_SealSegments_Call struct { } // SealSegments is a helper method to define mock.On call -// - ctx context.Context -// - channel string -// - segmentIDs []int64 +// - ctx context.Context +// - channel string +// - segmentIDs []int64 func (_e *MockBufferManager_Expecter) SealSegments(ctx interface{}, channel interface{}, segmentIDs interface{}) *MockBufferManager_SealSegments_Call { return &MockBufferManager_SealSegments_Call{Call: _e.mock.On("SealSegments", ctx, channel, segmentIDs)} } diff --git a/internal/mocks/mock_rootcoord.go b/internal/mocks/mock_rootcoord.go index 91274cd486..403c9e8fbc 100644 --- a/internal/mocks/mock_rootcoord.go +++ b/internal/mocks/mock_rootcoord.go @@ -68,8 +68,8 @@ type RootCoord_AllocID_Call struct { } // AllocID is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *rootcoordpb.AllocIDRequest +// - _a0 context.Context +// - _a1 *rootcoordpb.AllocIDRequest func (_e *RootCoord_Expecter) AllocID(_a0 interface{}, _a1 interface{}) *RootCoord_AllocID_Call { return &RootCoord_AllocID_Call{Call: _e.mock.On("AllocID", _a0, _a1)} } @@ -123,8 +123,8 @@ type RootCoord_AllocTimestamp_Call struct { } // AllocTimestamp is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *rootcoordpb.AllocTimestampRequest +// - _a0 context.Context +// - _a1 *rootcoordpb.AllocTimestampRequest func (_e *RootCoord_Expecter) AllocTimestamp(_a0 interface{}, _a1 interface{}) *RootCoord_AllocTimestamp_Call { return &RootCoord_AllocTimestamp_Call{Call: _e.mock.On("AllocTimestamp", _a0, _a1)} } @@ -178,8 +178,8 @@ type RootCoord_AlterAlias_Call struct { } // AlterAlias is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.AlterAliasRequest +// - _a0 context.Context +// - _a1 *milvuspb.AlterAliasRequest func (_e *RootCoord_Expecter) AlterAlias(_a0 interface{}, _a1 interface{}) *RootCoord_AlterAlias_Call { return &RootCoord_AlterAlias_Call{Call: _e.mock.On("AlterAlias", _a0, _a1)} } @@ -233,8 +233,8 @@ type RootCoord_AlterCollection_Call struct { } // AlterCollection is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.AlterCollectionRequest +// - _a0 context.Context +// - _a1 *milvuspb.AlterCollectionRequest func (_e *RootCoord_Expecter) AlterCollection(_a0 interface{}, _a1 interface{}) *RootCoord_AlterCollection_Call { return &RootCoord_AlterCollection_Call{Call: _e.mock.On("AlterCollection", _a0, _a1)} } @@ -288,8 +288,8 @@ type RootCoord_AlterDatabase_Call struct { } // AlterDatabase is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *rootcoordpb.AlterDatabaseRequest +// - _a0 context.Context +// - _a1 *rootcoordpb.AlterDatabaseRequest func (_e *RootCoord_Expecter) AlterDatabase(_a0 interface{}, _a1 interface{}) *RootCoord_AlterDatabase_Call { return &RootCoord_AlterDatabase_Call{Call: _e.mock.On("AlterDatabase", _a0, _a1)} } @@ -343,8 +343,8 @@ type RootCoord_CheckHealth_Call struct { } // CheckHealth is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.CheckHealthRequest +// - _a0 context.Context +// - _a1 *milvuspb.CheckHealthRequest func (_e *RootCoord_Expecter) CheckHealth(_a0 interface{}, _a1 interface{}) *RootCoord_CheckHealth_Call { return &RootCoord_CheckHealth_Call{Call: _e.mock.On("CheckHealth", _a0, _a1)} } @@ -398,8 +398,8 @@ type RootCoord_CreateAlias_Call struct { } // CreateAlias is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.CreateAliasRequest +// - _a0 context.Context +// - _a1 *milvuspb.CreateAliasRequest func (_e *RootCoord_Expecter) CreateAlias(_a0 interface{}, _a1 interface{}) *RootCoord_CreateAlias_Call { return &RootCoord_CreateAlias_Call{Call: _e.mock.On("CreateAlias", _a0, _a1)} } @@ -453,8 +453,8 @@ type RootCoord_CreateCollection_Call struct { } // CreateCollection is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.CreateCollectionRequest +// - _a0 context.Context +// - _a1 *milvuspb.CreateCollectionRequest func (_e *RootCoord_Expecter) CreateCollection(_a0 interface{}, _a1 interface{}) *RootCoord_CreateCollection_Call { return &RootCoord_CreateCollection_Call{Call: _e.mock.On("CreateCollection", _a0, _a1)} } @@ -508,8 +508,8 @@ type RootCoord_CreateCredential_Call struct { } // CreateCredential is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *internalpb.CredentialInfo +// - _a0 context.Context +// - _a1 *internalpb.CredentialInfo func (_e *RootCoord_Expecter) CreateCredential(_a0 interface{}, _a1 interface{}) *RootCoord_CreateCredential_Call { return &RootCoord_CreateCredential_Call{Call: _e.mock.On("CreateCredential", _a0, _a1)} } @@ -563,8 +563,8 @@ type RootCoord_CreateDatabase_Call struct { } // CreateDatabase is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.CreateDatabaseRequest +// - _a0 context.Context +// - _a1 *milvuspb.CreateDatabaseRequest func (_e *RootCoord_Expecter) CreateDatabase(_a0 interface{}, _a1 interface{}) *RootCoord_CreateDatabase_Call { return &RootCoord_CreateDatabase_Call{Call: _e.mock.On("CreateDatabase", _a0, _a1)} } @@ -618,8 +618,8 @@ type RootCoord_CreatePartition_Call struct { } // CreatePartition is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.CreatePartitionRequest +// - _a0 context.Context +// - _a1 *milvuspb.CreatePartitionRequest func (_e *RootCoord_Expecter) CreatePartition(_a0 interface{}, _a1 interface{}) *RootCoord_CreatePartition_Call { return &RootCoord_CreatePartition_Call{Call: _e.mock.On("CreatePartition", _a0, _a1)} } @@ -673,8 +673,8 @@ type RootCoord_CreateRole_Call struct { } // CreateRole is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.CreateRoleRequest +// - _a0 context.Context +// - _a1 *milvuspb.CreateRoleRequest func (_e *RootCoord_Expecter) CreateRole(_a0 interface{}, _a1 interface{}) *RootCoord_CreateRole_Call { return &RootCoord_CreateRole_Call{Call: _e.mock.On("CreateRole", _a0, _a1)} } @@ -728,8 +728,8 @@ type RootCoord_DeleteCredential_Call struct { } // DeleteCredential is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.DeleteCredentialRequest +// - _a0 context.Context +// - _a1 *milvuspb.DeleteCredentialRequest func (_e *RootCoord_Expecter) DeleteCredential(_a0 interface{}, _a1 interface{}) *RootCoord_DeleteCredential_Call { return &RootCoord_DeleteCredential_Call{Call: _e.mock.On("DeleteCredential", _a0, _a1)} } @@ -783,8 +783,8 @@ type RootCoord_DescribeAlias_Call struct { } // DescribeAlias is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.DescribeAliasRequest +// - _a0 context.Context +// - _a1 *milvuspb.DescribeAliasRequest func (_e *RootCoord_Expecter) DescribeAlias(_a0 interface{}, _a1 interface{}) *RootCoord_DescribeAlias_Call { return &RootCoord_DescribeAlias_Call{Call: _e.mock.On("DescribeAlias", _a0, _a1)} } @@ -838,8 +838,8 @@ type RootCoord_DescribeCollection_Call struct { } // DescribeCollection is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.DescribeCollectionRequest +// - _a0 context.Context +// - _a1 *milvuspb.DescribeCollectionRequest func (_e *RootCoord_Expecter) DescribeCollection(_a0 interface{}, _a1 interface{}) *RootCoord_DescribeCollection_Call { return &RootCoord_DescribeCollection_Call{Call: _e.mock.On("DescribeCollection", _a0, _a1)} } @@ -893,8 +893,8 @@ type RootCoord_DescribeCollectionInternal_Call struct { } // DescribeCollectionInternal is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.DescribeCollectionRequest +// - _a0 context.Context +// - _a1 *milvuspb.DescribeCollectionRequest func (_e *RootCoord_Expecter) DescribeCollectionInternal(_a0 interface{}, _a1 interface{}) *RootCoord_DescribeCollectionInternal_Call { return &RootCoord_DescribeCollectionInternal_Call{Call: _e.mock.On("DescribeCollectionInternal", _a0, _a1)} } @@ -948,8 +948,8 @@ type RootCoord_DescribeDatabase_Call struct { } // DescribeDatabase is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *rootcoordpb.DescribeDatabaseRequest +// - _a0 context.Context +// - _a1 *rootcoordpb.DescribeDatabaseRequest func (_e *RootCoord_Expecter) DescribeDatabase(_a0 interface{}, _a1 interface{}) *RootCoord_DescribeDatabase_Call { return &RootCoord_DescribeDatabase_Call{Call: _e.mock.On("DescribeDatabase", _a0, _a1)} } @@ -1003,8 +1003,8 @@ type RootCoord_DropAlias_Call struct { } // DropAlias is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.DropAliasRequest +// - _a0 context.Context +// - _a1 *milvuspb.DropAliasRequest func (_e *RootCoord_Expecter) DropAlias(_a0 interface{}, _a1 interface{}) *RootCoord_DropAlias_Call { return &RootCoord_DropAlias_Call{Call: _e.mock.On("DropAlias", _a0, _a1)} } @@ -1058,8 +1058,8 @@ type RootCoord_DropCollection_Call struct { } // DropCollection is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.DropCollectionRequest +// - _a0 context.Context +// - _a1 *milvuspb.DropCollectionRequest func (_e *RootCoord_Expecter) DropCollection(_a0 interface{}, _a1 interface{}) *RootCoord_DropCollection_Call { return &RootCoord_DropCollection_Call{Call: _e.mock.On("DropCollection", _a0, _a1)} } @@ -1113,8 +1113,8 @@ type RootCoord_DropDatabase_Call struct { } // DropDatabase is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.DropDatabaseRequest +// - _a0 context.Context +// - _a1 *milvuspb.DropDatabaseRequest func (_e *RootCoord_Expecter) DropDatabase(_a0 interface{}, _a1 interface{}) *RootCoord_DropDatabase_Call { return &RootCoord_DropDatabase_Call{Call: _e.mock.On("DropDatabase", _a0, _a1)} } @@ -1168,8 +1168,8 @@ type RootCoord_DropPartition_Call struct { } // DropPartition is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.DropPartitionRequest +// - _a0 context.Context +// - _a1 *milvuspb.DropPartitionRequest func (_e *RootCoord_Expecter) DropPartition(_a0 interface{}, _a1 interface{}) *RootCoord_DropPartition_Call { return &RootCoord_DropPartition_Call{Call: _e.mock.On("DropPartition", _a0, _a1)} } @@ -1223,8 +1223,8 @@ type RootCoord_DropRole_Call struct { } // DropRole is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.DropRoleRequest +// - _a0 context.Context +// - _a1 *milvuspb.DropRoleRequest func (_e *RootCoord_Expecter) DropRole(_a0 interface{}, _a1 interface{}) *RootCoord_DropRole_Call { return &RootCoord_DropRole_Call{Call: _e.mock.On("DropRole", _a0, _a1)} } @@ -1278,8 +1278,8 @@ type RootCoord_GetComponentStates_Call struct { } // GetComponentStates is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.GetComponentStatesRequest +// - _a0 context.Context +// - _a1 *milvuspb.GetComponentStatesRequest func (_e *RootCoord_Expecter) GetComponentStates(_a0 interface{}, _a1 interface{}) *RootCoord_GetComponentStates_Call { return &RootCoord_GetComponentStates_Call{Call: _e.mock.On("GetComponentStates", _a0, _a1)} } @@ -1333,8 +1333,8 @@ type RootCoord_GetCredential_Call struct { } // GetCredential is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *rootcoordpb.GetCredentialRequest +// - _a0 context.Context +// - _a1 *rootcoordpb.GetCredentialRequest func (_e *RootCoord_Expecter) GetCredential(_a0 interface{}, _a1 interface{}) *RootCoord_GetCredential_Call { return &RootCoord_GetCredential_Call{Call: _e.mock.On("GetCredential", _a0, _a1)} } @@ -1388,8 +1388,8 @@ type RootCoord_GetMetrics_Call struct { } // GetMetrics is a helper method to define mock.On call -// - ctx context.Context -// - req *milvuspb.GetMetricsRequest +// - ctx context.Context +// - req *milvuspb.GetMetricsRequest func (_e *RootCoord_Expecter) GetMetrics(ctx interface{}, req interface{}) *RootCoord_GetMetrics_Call { return &RootCoord_GetMetrics_Call{Call: _e.mock.On("GetMetrics", ctx, req)} } @@ -1443,8 +1443,8 @@ type RootCoord_GetStatisticsChannel_Call struct { } // GetStatisticsChannel is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *internalpb.GetStatisticsChannelRequest +// - _a0 context.Context +// - _a1 *internalpb.GetStatisticsChannelRequest func (_e *RootCoord_Expecter) GetStatisticsChannel(_a0 interface{}, _a1 interface{}) *RootCoord_GetStatisticsChannel_Call { return &RootCoord_GetStatisticsChannel_Call{Call: _e.mock.On("GetStatisticsChannel", _a0, _a1)} } @@ -1498,8 +1498,8 @@ type RootCoord_GetTimeTickChannel_Call struct { } // GetTimeTickChannel is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *internalpb.GetTimeTickChannelRequest +// - _a0 context.Context +// - _a1 *internalpb.GetTimeTickChannelRequest func (_e *RootCoord_Expecter) GetTimeTickChannel(_a0 interface{}, _a1 interface{}) *RootCoord_GetTimeTickChannel_Call { return &RootCoord_GetTimeTickChannel_Call{Call: _e.mock.On("GetTimeTickChannel", _a0, _a1)} } @@ -1553,8 +1553,8 @@ type RootCoord_GetVChannels_Call struct { } // GetVChannels is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *rootcoordpb.GetVChannelsRequest +// - _a0 context.Context +// - _a1 *rootcoordpb.GetVChannelsRequest func (_e *RootCoord_Expecter) GetVChannels(_a0 interface{}, _a1 interface{}) *RootCoord_GetVChannels_Call { return &RootCoord_GetVChannels_Call{Call: _e.mock.On("GetVChannels", _a0, _a1)} } @@ -1608,8 +1608,8 @@ type RootCoord_HasCollection_Call struct { } // HasCollection is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.HasCollectionRequest +// - _a0 context.Context +// - _a1 *milvuspb.HasCollectionRequest func (_e *RootCoord_Expecter) HasCollection(_a0 interface{}, _a1 interface{}) *RootCoord_HasCollection_Call { return &RootCoord_HasCollection_Call{Call: _e.mock.On("HasCollection", _a0, _a1)} } @@ -1663,8 +1663,8 @@ type RootCoord_HasPartition_Call struct { } // HasPartition is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.HasPartitionRequest +// - _a0 context.Context +// - _a1 *milvuspb.HasPartitionRequest func (_e *RootCoord_Expecter) HasPartition(_a0 interface{}, _a1 interface{}) *RootCoord_HasPartition_Call { return &RootCoord_HasPartition_Call{Call: _e.mock.On("HasPartition", _a0, _a1)} } @@ -1759,8 +1759,8 @@ type RootCoord_InvalidateCollectionMetaCache_Call struct { } // InvalidateCollectionMetaCache is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *proxypb.InvalidateCollMetaCacheRequest +// - _a0 context.Context +// - _a1 *proxypb.InvalidateCollMetaCacheRequest func (_e *RootCoord_Expecter) InvalidateCollectionMetaCache(_a0 interface{}, _a1 interface{}) *RootCoord_InvalidateCollectionMetaCache_Call { return &RootCoord_InvalidateCollectionMetaCache_Call{Call: _e.mock.On("InvalidateCollectionMetaCache", _a0, _a1)} } @@ -1814,8 +1814,8 @@ type RootCoord_ListAliases_Call struct { } // ListAliases is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.ListAliasesRequest +// - _a0 context.Context +// - _a1 *milvuspb.ListAliasesRequest func (_e *RootCoord_Expecter) ListAliases(_a0 interface{}, _a1 interface{}) *RootCoord_ListAliases_Call { return &RootCoord_ListAliases_Call{Call: _e.mock.On("ListAliases", _a0, _a1)} } @@ -1869,8 +1869,8 @@ type RootCoord_ListCredUsers_Call struct { } // ListCredUsers is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.ListCredUsersRequest +// - _a0 context.Context +// - _a1 *milvuspb.ListCredUsersRequest func (_e *RootCoord_Expecter) ListCredUsers(_a0 interface{}, _a1 interface{}) *RootCoord_ListCredUsers_Call { return &RootCoord_ListCredUsers_Call{Call: _e.mock.On("ListCredUsers", _a0, _a1)} } @@ -1924,8 +1924,8 @@ type RootCoord_ListDatabases_Call struct { } // ListDatabases is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.ListDatabasesRequest +// - _a0 context.Context +// - _a1 *milvuspb.ListDatabasesRequest func (_e *RootCoord_Expecter) ListDatabases(_a0 interface{}, _a1 interface{}) *RootCoord_ListDatabases_Call { return &RootCoord_ListDatabases_Call{Call: _e.mock.On("ListDatabases", _a0, _a1)} } @@ -1979,8 +1979,8 @@ type RootCoord_ListPolicy_Call struct { } // ListPolicy is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *internalpb.ListPolicyRequest +// - _a0 context.Context +// - _a1 *internalpb.ListPolicyRequest func (_e *RootCoord_Expecter) ListPolicy(_a0 interface{}, _a1 interface{}) *RootCoord_ListPolicy_Call { return &RootCoord_ListPolicy_Call{Call: _e.mock.On("ListPolicy", _a0, _a1)} } @@ -2034,8 +2034,8 @@ type RootCoord_OperatePrivilege_Call struct { } // OperatePrivilege is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.OperatePrivilegeRequest +// - _a0 context.Context +// - _a1 *milvuspb.OperatePrivilegeRequest func (_e *RootCoord_Expecter) OperatePrivilege(_a0 interface{}, _a1 interface{}) *RootCoord_OperatePrivilege_Call { return &RootCoord_OperatePrivilege_Call{Call: _e.mock.On("OperatePrivilege", _a0, _a1)} } @@ -2089,8 +2089,8 @@ type RootCoord_OperateUserRole_Call struct { } // OperateUserRole is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.OperateUserRoleRequest +// - _a0 context.Context +// - _a1 *milvuspb.OperateUserRoleRequest func (_e *RootCoord_Expecter) OperateUserRole(_a0 interface{}, _a1 interface{}) *RootCoord_OperateUserRole_Call { return &RootCoord_OperateUserRole_Call{Call: _e.mock.On("OperateUserRole", _a0, _a1)} } @@ -2185,8 +2185,8 @@ type RootCoord_RenameCollection_Call struct { } // RenameCollection is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.RenameCollectionRequest +// - _a0 context.Context +// - _a1 *milvuspb.RenameCollectionRequest func (_e *RootCoord_Expecter) RenameCollection(_a0 interface{}, _a1 interface{}) *RootCoord_RenameCollection_Call { return &RootCoord_RenameCollection_Call{Call: _e.mock.On("RenameCollection", _a0, _a1)} } @@ -2240,8 +2240,8 @@ type RootCoord_SelectGrant_Call struct { } // SelectGrant is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.SelectGrantRequest +// - _a0 context.Context +// - _a1 *milvuspb.SelectGrantRequest func (_e *RootCoord_Expecter) SelectGrant(_a0 interface{}, _a1 interface{}) *RootCoord_SelectGrant_Call { return &RootCoord_SelectGrant_Call{Call: _e.mock.On("SelectGrant", _a0, _a1)} } @@ -2295,8 +2295,8 @@ type RootCoord_SelectRole_Call struct { } // SelectRole is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.SelectRoleRequest +// - _a0 context.Context +// - _a1 *milvuspb.SelectRoleRequest func (_e *RootCoord_Expecter) SelectRole(_a0 interface{}, _a1 interface{}) *RootCoord_SelectRole_Call { return &RootCoord_SelectRole_Call{Call: _e.mock.On("SelectRole", _a0, _a1)} } @@ -2350,8 +2350,8 @@ type RootCoord_SelectUser_Call struct { } // SelectUser is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.SelectUserRequest +// - _a0 context.Context +// - _a1 *milvuspb.SelectUserRequest func (_e *RootCoord_Expecter) SelectUser(_a0 interface{}, _a1 interface{}) *RootCoord_SelectUser_Call { return &RootCoord_SelectUser_Call{Call: _e.mock.On("SelectUser", _a0, _a1)} } @@ -2384,7 +2384,7 @@ type RootCoord_SetAddress_Call struct { } // SetAddress is a helper method to define mock.On call -// - address string +// - address string func (_e *RootCoord_Expecter) SetAddress(address interface{}) *RootCoord_SetAddress_Call { return &RootCoord_SetAddress_Call{Call: _e.mock.On("SetAddress", address)} } @@ -2426,7 +2426,7 @@ type RootCoord_SetDataCoordClient_Call struct { } // SetDataCoordClient is a helper method to define mock.On call -// - dataCoord types.DataCoordClient +// - dataCoord types.DataCoordClient func (_e *RootCoord_Expecter) SetDataCoordClient(dataCoord interface{}) *RootCoord_SetDataCoordClient_Call { return &RootCoord_SetDataCoordClient_Call{Call: _e.mock.On("SetDataCoordClient", dataCoord)} } @@ -2459,7 +2459,7 @@ type RootCoord_SetEtcdClient_Call struct { } // SetEtcdClient is a helper method to define mock.On call -// - etcdClient *clientv3.Client +// - etcdClient *clientv3.Client func (_e *RootCoord_Expecter) SetEtcdClient(etcdClient interface{}) *RootCoord_SetEtcdClient_Call { return &RootCoord_SetEtcdClient_Call{Call: _e.mock.On("SetEtcdClient", etcdClient)} } @@ -2492,7 +2492,7 @@ type RootCoord_SetProxyCreator_Call struct { } // SetProxyCreator is a helper method to define mock.On call -// - _a0 func(context.Context , string , int64)(types.ProxyClient , error) +// - _a0 func(context.Context , string , int64)(types.ProxyClient , error) func (_e *RootCoord_Expecter) SetProxyCreator(_a0 interface{}) *RootCoord_SetProxyCreator_Call { return &RootCoord_SetProxyCreator_Call{Call: _e.mock.On("SetProxyCreator", _a0)} } @@ -2534,7 +2534,7 @@ type RootCoord_SetQueryCoordClient_Call struct { } // SetQueryCoordClient is a helper method to define mock.On call -// - queryCoord types.QueryCoordClient +// - queryCoord types.QueryCoordClient func (_e *RootCoord_Expecter) SetQueryCoordClient(queryCoord interface{}) *RootCoord_SetQueryCoordClient_Call { return &RootCoord_SetQueryCoordClient_Call{Call: _e.mock.On("SetQueryCoordClient", queryCoord)} } @@ -2567,7 +2567,7 @@ type RootCoord_SetTiKVClient_Call struct { } // SetTiKVClient is a helper method to define mock.On call -// - client *txnkv.Client +// - client *txnkv.Client func (_e *RootCoord_Expecter) SetTiKVClient(client interface{}) *RootCoord_SetTiKVClient_Call { return &RootCoord_SetTiKVClient_Call{Call: _e.mock.On("SetTiKVClient", client)} } @@ -2621,8 +2621,8 @@ type RootCoord_ShowCollections_Call struct { } // ShowCollections is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.ShowCollectionsRequest +// - _a0 context.Context +// - _a1 *milvuspb.ShowCollectionsRequest func (_e *RootCoord_Expecter) ShowCollections(_a0 interface{}, _a1 interface{}) *RootCoord_ShowCollections_Call { return &RootCoord_ShowCollections_Call{Call: _e.mock.On("ShowCollections", _a0, _a1)} } @@ -2676,8 +2676,8 @@ type RootCoord_ShowConfigurations_Call struct { } // ShowConfigurations is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *internalpb.ShowConfigurationsRequest +// - _a0 context.Context +// - _a1 *internalpb.ShowConfigurationsRequest func (_e *RootCoord_Expecter) ShowConfigurations(_a0 interface{}, _a1 interface{}) *RootCoord_ShowConfigurations_Call { return &RootCoord_ShowConfigurations_Call{Call: _e.mock.On("ShowConfigurations", _a0, _a1)} } @@ -2731,8 +2731,8 @@ type RootCoord_ShowPartitions_Call struct { } // ShowPartitions is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.ShowPartitionsRequest +// - _a0 context.Context +// - _a1 *milvuspb.ShowPartitionsRequest func (_e *RootCoord_Expecter) ShowPartitions(_a0 interface{}, _a1 interface{}) *RootCoord_ShowPartitions_Call { return &RootCoord_ShowPartitions_Call{Call: _e.mock.On("ShowPartitions", _a0, _a1)} } @@ -2786,8 +2786,8 @@ type RootCoord_ShowPartitionsInternal_Call struct { } // ShowPartitionsInternal is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.ShowPartitionsRequest +// - _a0 context.Context +// - _a1 *milvuspb.ShowPartitionsRequest func (_e *RootCoord_Expecter) ShowPartitionsInternal(_a0 interface{}, _a1 interface{}) *RootCoord_ShowPartitionsInternal_Call { return &RootCoord_ShowPartitionsInternal_Call{Call: _e.mock.On("ShowPartitionsInternal", _a0, _a1)} } @@ -2841,8 +2841,8 @@ type RootCoord_ShowSegments_Call struct { } // ShowSegments is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *milvuspb.ShowSegmentsRequest +// - _a0 context.Context +// - _a1 *milvuspb.ShowSegmentsRequest func (_e *RootCoord_Expecter) ShowSegments(_a0 interface{}, _a1 interface{}) *RootCoord_ShowSegments_Call { return &RootCoord_ShowSegments_Call{Call: _e.mock.On("ShowSegments", _a0, _a1)} } @@ -2978,8 +2978,8 @@ type RootCoord_UpdateChannelTimeTick_Call struct { } // UpdateChannelTimeTick is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *internalpb.ChannelTimeTickMsg +// - _a0 context.Context +// - _a1 *internalpb.ChannelTimeTickMsg func (_e *RootCoord_Expecter) UpdateChannelTimeTick(_a0 interface{}, _a1 interface{}) *RootCoord_UpdateChannelTimeTick_Call { return &RootCoord_UpdateChannelTimeTick_Call{Call: _e.mock.On("UpdateChannelTimeTick", _a0, _a1)} } @@ -3033,8 +3033,8 @@ type RootCoord_UpdateCredential_Call struct { } // UpdateCredential is a helper method to define mock.On call -// - _a0 context.Context -// - _a1 *internalpb.CredentialInfo +// - _a0 context.Context +// - _a1 *internalpb.CredentialInfo func (_e *RootCoord_Expecter) UpdateCredential(_a0 interface{}, _a1 interface{}) *RootCoord_UpdateCredential_Call { return &RootCoord_UpdateCredential_Call{Call: _e.mock.On("UpdateCredential", _a0, _a1)} } @@ -3067,7 +3067,7 @@ type RootCoord_UpdateStateCode_Call struct { } // UpdateStateCode is a helper method to define mock.On call -// - _a0 commonpb.StateCode +// - _a0 commonpb.StateCode func (_e *RootCoord_Expecter) UpdateStateCode(_a0 interface{}) *RootCoord_UpdateStateCode_Call { return &RootCoord_UpdateStateCode_Call{Call: _e.mock.On("UpdateStateCode", _a0)} } diff --git a/internal/mocks/mock_rootcoord_client.go b/internal/mocks/mock_rootcoord_client.go index 2c7f596b56..ffa3fdba17 100644 --- a/internal/mocks/mock_rootcoord_client.go +++ b/internal/mocks/mock_rootcoord_client.go @@ -72,9 +72,9 @@ type MockRootCoordClient_AllocID_Call struct { } // AllocID is a helper method to define mock.On call -// - ctx context.Context -// - in *rootcoordpb.AllocIDRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *rootcoordpb.AllocIDRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) AllocID(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_AllocID_Call { return &MockRootCoordClient_AllocID_Call{Call: _e.mock.On("AllocID", append([]interface{}{ctx, in}, opts...)...)} @@ -142,9 +142,9 @@ type MockRootCoordClient_AllocTimestamp_Call struct { } // AllocTimestamp is a helper method to define mock.On call -// - ctx context.Context -// - in *rootcoordpb.AllocTimestampRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *rootcoordpb.AllocTimestampRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) AllocTimestamp(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_AllocTimestamp_Call { return &MockRootCoordClient_AllocTimestamp_Call{Call: _e.mock.On("AllocTimestamp", append([]interface{}{ctx, in}, opts...)...)} @@ -212,9 +212,9 @@ type MockRootCoordClient_AlterAlias_Call struct { } // AlterAlias is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.AlterAliasRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.AlterAliasRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) AlterAlias(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_AlterAlias_Call { return &MockRootCoordClient_AlterAlias_Call{Call: _e.mock.On("AlterAlias", append([]interface{}{ctx, in}, opts...)...)} @@ -282,9 +282,9 @@ type MockRootCoordClient_AlterCollection_Call struct { } // AlterCollection is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.AlterCollectionRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.AlterCollectionRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) AlterCollection(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_AlterCollection_Call { return &MockRootCoordClient_AlterCollection_Call{Call: _e.mock.On("AlterCollection", append([]interface{}{ctx, in}, opts...)...)} @@ -352,9 +352,9 @@ type MockRootCoordClient_AlterDatabase_Call struct { } // AlterDatabase is a helper method to define mock.On call -// - ctx context.Context -// - in *rootcoordpb.AlterDatabaseRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *rootcoordpb.AlterDatabaseRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) AlterDatabase(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_AlterDatabase_Call { return &MockRootCoordClient_AlterDatabase_Call{Call: _e.mock.On("AlterDatabase", append([]interface{}{ctx, in}, opts...)...)} @@ -422,9 +422,9 @@ type MockRootCoordClient_CheckHealth_Call struct { } // CheckHealth is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.CheckHealthRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.CheckHealthRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) CheckHealth(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_CheckHealth_Call { return &MockRootCoordClient_CheckHealth_Call{Call: _e.mock.On("CheckHealth", append([]interface{}{ctx, in}, opts...)...)} @@ -533,9 +533,9 @@ type MockRootCoordClient_CreateAlias_Call struct { } // CreateAlias is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.CreateAliasRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.CreateAliasRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) CreateAlias(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_CreateAlias_Call { return &MockRootCoordClient_CreateAlias_Call{Call: _e.mock.On("CreateAlias", append([]interface{}{ctx, in}, opts...)...)} @@ -603,9 +603,9 @@ type MockRootCoordClient_CreateCollection_Call struct { } // CreateCollection is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.CreateCollectionRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.CreateCollectionRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) CreateCollection(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_CreateCollection_Call { return &MockRootCoordClient_CreateCollection_Call{Call: _e.mock.On("CreateCollection", append([]interface{}{ctx, in}, opts...)...)} @@ -673,9 +673,9 @@ type MockRootCoordClient_CreateCredential_Call struct { } // CreateCredential is a helper method to define mock.On call -// - ctx context.Context -// - in *internalpb.CredentialInfo -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *internalpb.CredentialInfo +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) CreateCredential(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_CreateCredential_Call { return &MockRootCoordClient_CreateCredential_Call{Call: _e.mock.On("CreateCredential", append([]interface{}{ctx, in}, opts...)...)} @@ -743,9 +743,9 @@ type MockRootCoordClient_CreateDatabase_Call struct { } // CreateDatabase is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.CreateDatabaseRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.CreateDatabaseRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) CreateDatabase(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_CreateDatabase_Call { return &MockRootCoordClient_CreateDatabase_Call{Call: _e.mock.On("CreateDatabase", append([]interface{}{ctx, in}, opts...)...)} @@ -813,9 +813,9 @@ type MockRootCoordClient_CreatePartition_Call struct { } // CreatePartition is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.CreatePartitionRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.CreatePartitionRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) CreatePartition(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_CreatePartition_Call { return &MockRootCoordClient_CreatePartition_Call{Call: _e.mock.On("CreatePartition", append([]interface{}{ctx, in}, opts...)...)} @@ -883,9 +883,9 @@ type MockRootCoordClient_CreateRole_Call struct { } // CreateRole is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.CreateRoleRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.CreateRoleRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) CreateRole(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_CreateRole_Call { return &MockRootCoordClient_CreateRole_Call{Call: _e.mock.On("CreateRole", append([]interface{}{ctx, in}, opts...)...)} @@ -953,9 +953,9 @@ type MockRootCoordClient_DeleteCredential_Call struct { } // DeleteCredential is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.DeleteCredentialRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.DeleteCredentialRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) DeleteCredential(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_DeleteCredential_Call { return &MockRootCoordClient_DeleteCredential_Call{Call: _e.mock.On("DeleteCredential", append([]interface{}{ctx, in}, opts...)...)} @@ -1023,9 +1023,9 @@ type MockRootCoordClient_DescribeAlias_Call struct { } // DescribeAlias is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.DescribeAliasRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.DescribeAliasRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) DescribeAlias(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_DescribeAlias_Call { return &MockRootCoordClient_DescribeAlias_Call{Call: _e.mock.On("DescribeAlias", append([]interface{}{ctx, in}, opts...)...)} @@ -1093,9 +1093,9 @@ type MockRootCoordClient_DescribeCollection_Call struct { } // DescribeCollection is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.DescribeCollectionRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.DescribeCollectionRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) DescribeCollection(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_DescribeCollection_Call { return &MockRootCoordClient_DescribeCollection_Call{Call: _e.mock.On("DescribeCollection", append([]interface{}{ctx, in}, opts...)...)} @@ -1163,9 +1163,9 @@ type MockRootCoordClient_DescribeCollectionInternal_Call struct { } // DescribeCollectionInternal is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.DescribeCollectionRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.DescribeCollectionRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) DescribeCollectionInternal(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_DescribeCollectionInternal_Call { return &MockRootCoordClient_DescribeCollectionInternal_Call{Call: _e.mock.On("DescribeCollectionInternal", append([]interface{}{ctx, in}, opts...)...)} @@ -1233,9 +1233,9 @@ type MockRootCoordClient_DescribeDatabase_Call struct { } // DescribeDatabase is a helper method to define mock.On call -// - ctx context.Context -// - in *rootcoordpb.DescribeDatabaseRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *rootcoordpb.DescribeDatabaseRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) DescribeDatabase(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_DescribeDatabase_Call { return &MockRootCoordClient_DescribeDatabase_Call{Call: _e.mock.On("DescribeDatabase", append([]interface{}{ctx, in}, opts...)...)} @@ -1303,9 +1303,9 @@ type MockRootCoordClient_DropAlias_Call struct { } // DropAlias is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.DropAliasRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.DropAliasRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) DropAlias(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_DropAlias_Call { return &MockRootCoordClient_DropAlias_Call{Call: _e.mock.On("DropAlias", append([]interface{}{ctx, in}, opts...)...)} @@ -1373,9 +1373,9 @@ type MockRootCoordClient_DropCollection_Call struct { } // DropCollection is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.DropCollectionRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.DropCollectionRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) DropCollection(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_DropCollection_Call { return &MockRootCoordClient_DropCollection_Call{Call: _e.mock.On("DropCollection", append([]interface{}{ctx, in}, opts...)...)} @@ -1443,9 +1443,9 @@ type MockRootCoordClient_DropDatabase_Call struct { } // DropDatabase is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.DropDatabaseRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.DropDatabaseRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) DropDatabase(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_DropDatabase_Call { return &MockRootCoordClient_DropDatabase_Call{Call: _e.mock.On("DropDatabase", append([]interface{}{ctx, in}, opts...)...)} @@ -1513,9 +1513,9 @@ type MockRootCoordClient_DropPartition_Call struct { } // DropPartition is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.DropPartitionRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.DropPartitionRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) DropPartition(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_DropPartition_Call { return &MockRootCoordClient_DropPartition_Call{Call: _e.mock.On("DropPartition", append([]interface{}{ctx, in}, opts...)...)} @@ -1583,9 +1583,9 @@ type MockRootCoordClient_DropRole_Call struct { } // DropRole is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.DropRoleRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.DropRoleRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) DropRole(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_DropRole_Call { return &MockRootCoordClient_DropRole_Call{Call: _e.mock.On("DropRole", append([]interface{}{ctx, in}, opts...)...)} @@ -1653,9 +1653,9 @@ type MockRootCoordClient_GetComponentStates_Call struct { } // GetComponentStates is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.GetComponentStatesRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.GetComponentStatesRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) GetComponentStates(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_GetComponentStates_Call { return &MockRootCoordClient_GetComponentStates_Call{Call: _e.mock.On("GetComponentStates", append([]interface{}{ctx, in}, opts...)...)} @@ -1723,9 +1723,9 @@ type MockRootCoordClient_GetCredential_Call struct { } // GetCredential is a helper method to define mock.On call -// - ctx context.Context -// - in *rootcoordpb.GetCredentialRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *rootcoordpb.GetCredentialRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) GetCredential(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_GetCredential_Call { return &MockRootCoordClient_GetCredential_Call{Call: _e.mock.On("GetCredential", append([]interface{}{ctx, in}, opts...)...)} @@ -1793,9 +1793,9 @@ type MockRootCoordClient_GetMetrics_Call struct { } // GetMetrics is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.GetMetricsRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.GetMetricsRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) GetMetrics(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_GetMetrics_Call { return &MockRootCoordClient_GetMetrics_Call{Call: _e.mock.On("GetMetrics", append([]interface{}{ctx, in}, opts...)...)} @@ -1863,9 +1863,9 @@ type MockRootCoordClient_GetStatisticsChannel_Call struct { } // GetStatisticsChannel is a helper method to define mock.On call -// - ctx context.Context -// - in *internalpb.GetStatisticsChannelRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *internalpb.GetStatisticsChannelRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) GetStatisticsChannel(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_GetStatisticsChannel_Call { return &MockRootCoordClient_GetStatisticsChannel_Call{Call: _e.mock.On("GetStatisticsChannel", append([]interface{}{ctx, in}, opts...)...)} @@ -1933,9 +1933,9 @@ type MockRootCoordClient_GetTimeTickChannel_Call struct { } // GetTimeTickChannel is a helper method to define mock.On call -// - ctx context.Context -// - in *internalpb.GetTimeTickChannelRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *internalpb.GetTimeTickChannelRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) GetTimeTickChannel(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_GetTimeTickChannel_Call { return &MockRootCoordClient_GetTimeTickChannel_Call{Call: _e.mock.On("GetTimeTickChannel", append([]interface{}{ctx, in}, opts...)...)} @@ -2003,9 +2003,9 @@ type MockRootCoordClient_GetVChannels_Call struct { } // GetVChannels is a helper method to define mock.On call -// - ctx context.Context -// - in *rootcoordpb.GetVChannelsRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *rootcoordpb.GetVChannelsRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) GetVChannels(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_GetVChannels_Call { return &MockRootCoordClient_GetVChannels_Call{Call: _e.mock.On("GetVChannels", append([]interface{}{ctx, in}, opts...)...)} @@ -2073,9 +2073,9 @@ type MockRootCoordClient_HasCollection_Call struct { } // HasCollection is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.HasCollectionRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.HasCollectionRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) HasCollection(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_HasCollection_Call { return &MockRootCoordClient_HasCollection_Call{Call: _e.mock.On("HasCollection", append([]interface{}{ctx, in}, opts...)...)} @@ -2143,9 +2143,9 @@ type MockRootCoordClient_HasPartition_Call struct { } // HasPartition is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.HasPartitionRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.HasPartitionRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) HasPartition(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_HasPartition_Call { return &MockRootCoordClient_HasPartition_Call{Call: _e.mock.On("HasPartition", append([]interface{}{ctx, in}, opts...)...)} @@ -2213,9 +2213,9 @@ type MockRootCoordClient_InvalidateCollectionMetaCache_Call struct { } // InvalidateCollectionMetaCache is a helper method to define mock.On call -// - ctx context.Context -// - in *proxypb.InvalidateCollMetaCacheRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *proxypb.InvalidateCollMetaCacheRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) InvalidateCollectionMetaCache(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_InvalidateCollectionMetaCache_Call { return &MockRootCoordClient_InvalidateCollectionMetaCache_Call{Call: _e.mock.On("InvalidateCollectionMetaCache", append([]interface{}{ctx, in}, opts...)...)} @@ -2283,9 +2283,9 @@ type MockRootCoordClient_ListAliases_Call struct { } // ListAliases is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.ListAliasesRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.ListAliasesRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) ListAliases(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_ListAliases_Call { return &MockRootCoordClient_ListAliases_Call{Call: _e.mock.On("ListAliases", append([]interface{}{ctx, in}, opts...)...)} @@ -2353,9 +2353,9 @@ type MockRootCoordClient_ListCredUsers_Call struct { } // ListCredUsers is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.ListCredUsersRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.ListCredUsersRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) ListCredUsers(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_ListCredUsers_Call { return &MockRootCoordClient_ListCredUsers_Call{Call: _e.mock.On("ListCredUsers", append([]interface{}{ctx, in}, opts...)...)} @@ -2423,9 +2423,9 @@ type MockRootCoordClient_ListDatabases_Call struct { } // ListDatabases is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.ListDatabasesRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.ListDatabasesRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) ListDatabases(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_ListDatabases_Call { return &MockRootCoordClient_ListDatabases_Call{Call: _e.mock.On("ListDatabases", append([]interface{}{ctx, in}, opts...)...)} @@ -2493,9 +2493,9 @@ type MockRootCoordClient_ListPolicy_Call struct { } // ListPolicy is a helper method to define mock.On call -// - ctx context.Context -// - in *internalpb.ListPolicyRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *internalpb.ListPolicyRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) ListPolicy(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_ListPolicy_Call { return &MockRootCoordClient_ListPolicy_Call{Call: _e.mock.On("ListPolicy", append([]interface{}{ctx, in}, opts...)...)} @@ -2563,9 +2563,9 @@ type MockRootCoordClient_OperatePrivilege_Call struct { } // OperatePrivilege is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.OperatePrivilegeRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.OperatePrivilegeRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) OperatePrivilege(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_OperatePrivilege_Call { return &MockRootCoordClient_OperatePrivilege_Call{Call: _e.mock.On("OperatePrivilege", append([]interface{}{ctx, in}, opts...)...)} @@ -2633,9 +2633,9 @@ type MockRootCoordClient_OperateUserRole_Call struct { } // OperateUserRole is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.OperateUserRoleRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.OperateUserRoleRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) OperateUserRole(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_OperateUserRole_Call { return &MockRootCoordClient_OperateUserRole_Call{Call: _e.mock.On("OperateUserRole", append([]interface{}{ctx, in}, opts...)...)} @@ -2703,9 +2703,9 @@ type MockRootCoordClient_RenameCollection_Call struct { } // RenameCollection is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.RenameCollectionRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.RenameCollectionRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) RenameCollection(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_RenameCollection_Call { return &MockRootCoordClient_RenameCollection_Call{Call: _e.mock.On("RenameCollection", append([]interface{}{ctx, in}, opts...)...)} @@ -2773,9 +2773,9 @@ type MockRootCoordClient_SelectGrant_Call struct { } // SelectGrant is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.SelectGrantRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.SelectGrantRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) SelectGrant(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_SelectGrant_Call { return &MockRootCoordClient_SelectGrant_Call{Call: _e.mock.On("SelectGrant", append([]interface{}{ctx, in}, opts...)...)} @@ -2843,9 +2843,9 @@ type MockRootCoordClient_SelectRole_Call struct { } // SelectRole is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.SelectRoleRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.SelectRoleRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) SelectRole(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_SelectRole_Call { return &MockRootCoordClient_SelectRole_Call{Call: _e.mock.On("SelectRole", append([]interface{}{ctx, in}, opts...)...)} @@ -2913,9 +2913,9 @@ type MockRootCoordClient_SelectUser_Call struct { } // SelectUser is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.SelectUserRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.SelectUserRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) SelectUser(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_SelectUser_Call { return &MockRootCoordClient_SelectUser_Call{Call: _e.mock.On("SelectUser", append([]interface{}{ctx, in}, opts...)...)} @@ -2983,9 +2983,9 @@ type MockRootCoordClient_ShowCollections_Call struct { } // ShowCollections is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.ShowCollectionsRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.ShowCollectionsRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) ShowCollections(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_ShowCollections_Call { return &MockRootCoordClient_ShowCollections_Call{Call: _e.mock.On("ShowCollections", append([]interface{}{ctx, in}, opts...)...)} @@ -3053,9 +3053,9 @@ type MockRootCoordClient_ShowConfigurations_Call struct { } // ShowConfigurations is a helper method to define mock.On call -// - ctx context.Context -// - in *internalpb.ShowConfigurationsRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *internalpb.ShowConfigurationsRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) ShowConfigurations(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_ShowConfigurations_Call { return &MockRootCoordClient_ShowConfigurations_Call{Call: _e.mock.On("ShowConfigurations", append([]interface{}{ctx, in}, opts...)...)} @@ -3123,9 +3123,9 @@ type MockRootCoordClient_ShowPartitions_Call struct { } // ShowPartitions is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.ShowPartitionsRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.ShowPartitionsRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) ShowPartitions(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_ShowPartitions_Call { return &MockRootCoordClient_ShowPartitions_Call{Call: _e.mock.On("ShowPartitions", append([]interface{}{ctx, in}, opts...)...)} @@ -3193,9 +3193,9 @@ type MockRootCoordClient_ShowPartitionsInternal_Call struct { } // ShowPartitionsInternal is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.ShowPartitionsRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.ShowPartitionsRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) ShowPartitionsInternal(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_ShowPartitionsInternal_Call { return &MockRootCoordClient_ShowPartitionsInternal_Call{Call: _e.mock.On("ShowPartitionsInternal", append([]interface{}{ctx, in}, opts...)...)} @@ -3263,9 +3263,9 @@ type MockRootCoordClient_ShowSegments_Call struct { } // ShowSegments is a helper method to define mock.On call -// - ctx context.Context -// - in *milvuspb.ShowSegmentsRequest -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *milvuspb.ShowSegmentsRequest +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) ShowSegments(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_ShowSegments_Call { return &MockRootCoordClient_ShowSegments_Call{Call: _e.mock.On("ShowSegments", append([]interface{}{ctx, in}, opts...)...)} @@ -3333,9 +3333,9 @@ type MockRootCoordClient_UpdateChannelTimeTick_Call struct { } // UpdateChannelTimeTick is a helper method to define mock.On call -// - ctx context.Context -// - in *internalpb.ChannelTimeTickMsg -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *internalpb.ChannelTimeTickMsg +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) UpdateChannelTimeTick(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_UpdateChannelTimeTick_Call { return &MockRootCoordClient_UpdateChannelTimeTick_Call{Call: _e.mock.On("UpdateChannelTimeTick", append([]interface{}{ctx, in}, opts...)...)} @@ -3403,9 +3403,9 @@ type MockRootCoordClient_UpdateCredential_Call struct { } // UpdateCredential is a helper method to define mock.On call -// - ctx context.Context -// - in *internalpb.CredentialInfo -// - opts ...grpc.CallOption +// - ctx context.Context +// - in *internalpb.CredentialInfo +// - opts ...grpc.CallOption func (_e *MockRootCoordClient_Expecter) UpdateCredential(ctx interface{}, in interface{}, opts ...interface{}) *MockRootCoordClient_UpdateCredential_Call { return &MockRootCoordClient_UpdateCredential_Call{Call: _e.mock.On("UpdateCredential", append([]interface{}{ctx, in}, opts...)...)} diff --git a/internal/rootcoord/mocks/meta_table.go b/internal/rootcoord/mocks/meta_table.go index 73619b9664..e982b39d47 100644 --- a/internal/rootcoord/mocks/meta_table.go +++ b/internal/rootcoord/mocks/meta_table.go @@ -48,8 +48,8 @@ type IMetaTable_AddCollection_Call struct { } // AddCollection is a helper method to define mock.On call -// - ctx context.Context -// - coll *model.Collection +// - ctx context.Context +// - coll *model.Collection func (_e *IMetaTable_Expecter) AddCollection(ctx interface{}, coll interface{}) *IMetaTable_AddCollection_Call { return &IMetaTable_AddCollection_Call{Call: _e.mock.On("AddCollection", ctx, coll)} } @@ -91,7 +91,7 @@ type IMetaTable_AddCredential_Call struct { } // AddCredential is a helper method to define mock.On call -// - credInfo *internalpb.CredentialInfo +// - credInfo *internalpb.CredentialInfo func (_e *IMetaTable_Expecter) AddCredential(credInfo interface{}) *IMetaTable_AddCredential_Call { return &IMetaTable_AddCredential_Call{Call: _e.mock.On("AddCredential", credInfo)} } @@ -133,8 +133,8 @@ type IMetaTable_AddPartition_Call struct { } // AddPartition is a helper method to define mock.On call -// - ctx context.Context -// - partition *model.Partition +// - ctx context.Context +// - partition *model.Partition func (_e *IMetaTable_Expecter) AddPartition(ctx interface{}, partition interface{}) *IMetaTable_AddPartition_Call { return &IMetaTable_AddPartition_Call{Call: _e.mock.On("AddPartition", ctx, partition)} } @@ -176,11 +176,11 @@ type IMetaTable_AlterAlias_Call struct { } // AlterAlias is a helper method to define mock.On call -// - ctx context.Context -// - dbName string -// - alias string -// - collectionName string -// - ts uint64 +// - ctx context.Context +// - dbName string +// - alias string +// - collectionName string +// - ts uint64 func (_e *IMetaTable_Expecter) AlterAlias(ctx interface{}, dbName interface{}, alias interface{}, collectionName interface{}, ts interface{}) *IMetaTable_AlterAlias_Call { return &IMetaTable_AlterAlias_Call{Call: _e.mock.On("AlterAlias", ctx, dbName, alias, collectionName, ts)} } @@ -222,10 +222,10 @@ type IMetaTable_AlterCollection_Call struct { } // AlterCollection is a helper method to define mock.On call -// - ctx context.Context -// - oldColl *model.Collection -// - newColl *model.Collection -// - ts uint64 +// - ctx context.Context +// - oldColl *model.Collection +// - newColl *model.Collection +// - ts uint64 func (_e *IMetaTable_Expecter) AlterCollection(ctx interface{}, oldColl interface{}, newColl interface{}, ts interface{}) *IMetaTable_AlterCollection_Call { return &IMetaTable_AlterCollection_Call{Call: _e.mock.On("AlterCollection", ctx, oldColl, newColl, ts)} } @@ -267,7 +267,7 @@ type IMetaTable_AlterCredential_Call struct { } // AlterCredential is a helper method to define mock.On call -// - credInfo *internalpb.CredentialInfo +// - credInfo *internalpb.CredentialInfo func (_e *IMetaTable_Expecter) AlterCredential(credInfo interface{}) *IMetaTable_AlterCredential_Call { return &IMetaTable_AlterCredential_Call{Call: _e.mock.On("AlterCredential", credInfo)} } @@ -309,10 +309,10 @@ type IMetaTable_AlterDatabase_Call struct { } // AlterDatabase is a helper method to define mock.On call -// - ctx context.Context -// - oldDB *model.Database -// - newDB *model.Database -// - ts uint64 +// - ctx context.Context +// - oldDB *model.Database +// - newDB *model.Database +// - ts uint64 func (_e *IMetaTable_Expecter) AlterDatabase(ctx interface{}, oldDB interface{}, newDB interface{}, ts interface{}) *IMetaTable_AlterDatabase_Call { return &IMetaTable_AlterDatabase_Call{Call: _e.mock.On("AlterDatabase", ctx, oldDB, newDB, ts)} } @@ -354,10 +354,10 @@ type IMetaTable_ChangeCollectionState_Call struct { } // ChangeCollectionState is a helper method to define mock.On call -// - ctx context.Context -// - collectionID int64 -// - state etcdpb.CollectionState -// - ts uint64 +// - ctx context.Context +// - collectionID int64 +// - state etcdpb.CollectionState +// - ts uint64 func (_e *IMetaTable_Expecter) ChangeCollectionState(ctx interface{}, collectionID interface{}, state interface{}, ts interface{}) *IMetaTable_ChangeCollectionState_Call { return &IMetaTable_ChangeCollectionState_Call{Call: _e.mock.On("ChangeCollectionState", ctx, collectionID, state, ts)} } @@ -399,11 +399,11 @@ type IMetaTable_ChangePartitionState_Call struct { } // ChangePartitionState is a helper method to define mock.On call -// - ctx context.Context -// - collectionID int64 -// - partitionID int64 -// - state etcdpb.PartitionState -// - ts uint64 +// - ctx context.Context +// - collectionID int64 +// - partitionID int64 +// - state etcdpb.PartitionState +// - ts uint64 func (_e *IMetaTable_Expecter) ChangePartitionState(ctx interface{}, collectionID interface{}, partitionID interface{}, state interface{}, ts interface{}) *IMetaTable_ChangePartitionState_Call { return &IMetaTable_ChangePartitionState_Call{Call: _e.mock.On("ChangePartitionState", ctx, collectionID, partitionID, state, ts)} } @@ -445,11 +445,11 @@ type IMetaTable_CreateAlias_Call struct { } // CreateAlias is a helper method to define mock.On call -// - ctx context.Context -// - dbName string -// - alias string -// - collectionName string -// - ts uint64 +// - ctx context.Context +// - dbName string +// - alias string +// - collectionName string +// - ts uint64 func (_e *IMetaTable_Expecter) CreateAlias(ctx interface{}, dbName interface{}, alias interface{}, collectionName interface{}, ts interface{}) *IMetaTable_CreateAlias_Call { return &IMetaTable_CreateAlias_Call{Call: _e.mock.On("CreateAlias", ctx, dbName, alias, collectionName, ts)} } @@ -491,9 +491,9 @@ type IMetaTable_CreateDatabase_Call struct { } // CreateDatabase is a helper method to define mock.On call -// - ctx context.Context -// - db *model.Database -// - ts uint64 +// - ctx context.Context +// - db *model.Database +// - ts uint64 func (_e *IMetaTable_Expecter) CreateDatabase(ctx interface{}, db interface{}, ts interface{}) *IMetaTable_CreateDatabase_Call { return &IMetaTable_CreateDatabase_Call{Call: _e.mock.On("CreateDatabase", ctx, db, ts)} } @@ -535,8 +535,8 @@ type IMetaTable_CreateRole_Call struct { } // CreateRole is a helper method to define mock.On call -// - tenant string -// - entity *milvuspb.RoleEntity +// - tenant string +// - entity *milvuspb.RoleEntity func (_e *IMetaTable_Expecter) CreateRole(tenant interface{}, entity interface{}) *IMetaTable_CreateRole_Call { return &IMetaTable_CreateRole_Call{Call: _e.mock.On("CreateRole", tenant, entity)} } @@ -578,7 +578,7 @@ type IMetaTable_DeleteCredential_Call struct { } // DeleteCredential is a helper method to define mock.On call -// - username string +// - username string func (_e *IMetaTable_Expecter) DeleteCredential(username interface{}) *IMetaTable_DeleteCredential_Call { return &IMetaTable_DeleteCredential_Call{Call: _e.mock.On("DeleteCredential", username)} } @@ -630,10 +630,10 @@ type IMetaTable_DescribeAlias_Call struct { } // DescribeAlias is a helper method to define mock.On call -// - ctx context.Context -// - dbName string -// - alias string -// - ts uint64 +// - ctx context.Context +// - dbName string +// - alias string +// - ts uint64 func (_e *IMetaTable_Expecter) DescribeAlias(ctx interface{}, dbName interface{}, alias interface{}, ts interface{}) *IMetaTable_DescribeAlias_Call { return &IMetaTable_DescribeAlias_Call{Call: _e.mock.On("DescribeAlias", ctx, dbName, alias, ts)} } @@ -675,10 +675,10 @@ type IMetaTable_DropAlias_Call struct { } // DropAlias is a helper method to define mock.On call -// - ctx context.Context -// - dbName string -// - alias string -// - ts uint64 +// - ctx context.Context +// - dbName string +// - alias string +// - ts uint64 func (_e *IMetaTable_Expecter) DropAlias(ctx interface{}, dbName interface{}, alias interface{}, ts interface{}) *IMetaTable_DropAlias_Call { return &IMetaTable_DropAlias_Call{Call: _e.mock.On("DropAlias", ctx, dbName, alias, ts)} } @@ -720,9 +720,9 @@ type IMetaTable_DropDatabase_Call struct { } // DropDatabase is a helper method to define mock.On call -// - ctx context.Context -// - dbName string -// - ts uint64 +// - ctx context.Context +// - dbName string +// - ts uint64 func (_e *IMetaTable_Expecter) DropDatabase(ctx interface{}, dbName interface{}, ts interface{}) *IMetaTable_DropDatabase_Call { return &IMetaTable_DropDatabase_Call{Call: _e.mock.On("DropDatabase", ctx, dbName, ts)} } @@ -764,8 +764,8 @@ type IMetaTable_DropGrant_Call struct { } // DropGrant is a helper method to define mock.On call -// - tenant string -// - role *milvuspb.RoleEntity +// - tenant string +// - role *milvuspb.RoleEntity func (_e *IMetaTable_Expecter) DropGrant(tenant interface{}, role interface{}) *IMetaTable_DropGrant_Call { return &IMetaTable_DropGrant_Call{Call: _e.mock.On("DropGrant", tenant, role)} } @@ -807,8 +807,8 @@ type IMetaTable_DropRole_Call struct { } // DropRole is a helper method to define mock.On call -// - tenant string -// - roleName string +// - tenant string +// - roleName string func (_e *IMetaTable_Expecter) DropRole(tenant interface{}, roleName interface{}) *IMetaTable_DropRole_Call { return &IMetaTable_DropRole_Call{Call: _e.mock.On("DropRole", tenant, roleName)} } @@ -862,11 +862,11 @@ type IMetaTable_GetCollectionByID_Call struct { } // GetCollectionByID is a helper method to define mock.On call -// - ctx context.Context -// - dbName string -// - collectionID int64 -// - ts uint64 -// - allowUnavailable bool +// - ctx context.Context +// - dbName string +// - collectionID int64 +// - ts uint64 +// - allowUnavailable bool func (_e *IMetaTable_Expecter) GetCollectionByID(ctx interface{}, dbName interface{}, collectionID interface{}, ts interface{}, allowUnavailable interface{}) *IMetaTable_GetCollectionByID_Call { return &IMetaTable_GetCollectionByID_Call{Call: _e.mock.On("GetCollectionByID", ctx, dbName, collectionID, ts, allowUnavailable)} } @@ -920,8 +920,8 @@ type IMetaTable_GetCollectionByIDWithMaxTs_Call struct { } // GetCollectionByIDWithMaxTs is a helper method to define mock.On call -// - ctx context.Context -// - collectionID int64 +// - ctx context.Context +// - collectionID int64 func (_e *IMetaTable_Expecter) GetCollectionByIDWithMaxTs(ctx interface{}, collectionID interface{}) *IMetaTable_GetCollectionByIDWithMaxTs_Call { return &IMetaTable_GetCollectionByIDWithMaxTs_Call{Call: _e.mock.On("GetCollectionByIDWithMaxTs", ctx, collectionID)} } @@ -975,10 +975,10 @@ type IMetaTable_GetCollectionByName_Call struct { } // GetCollectionByName is a helper method to define mock.On call -// - ctx context.Context -// - dbName string -// - collectionName string -// - ts uint64 +// - ctx context.Context +// - dbName string +// - collectionName string +// - ts uint64 func (_e *IMetaTable_Expecter) GetCollectionByName(ctx interface{}, dbName interface{}, collectionName interface{}, ts interface{}) *IMetaTable_GetCollectionByName_Call { return &IMetaTable_GetCollectionByName_Call{Call: _e.mock.On("GetCollectionByName", ctx, dbName, collectionName, ts)} } @@ -1022,7 +1022,7 @@ type IMetaTable_GetCollectionVirtualChannels_Call struct { } // GetCollectionVirtualChannels is a helper method to define mock.On call -// - colID int64 +// - colID int64 func (_e *IMetaTable_Expecter) GetCollectionVirtualChannels(colID interface{}) *IMetaTable_GetCollectionVirtualChannels_Call { return &IMetaTable_GetCollectionVirtualChannels_Call{Call: _e.mock.On("GetCollectionVirtualChannels", colID)} } @@ -1076,7 +1076,7 @@ type IMetaTable_GetCredential_Call struct { } // GetCredential is a helper method to define mock.On call -// - username string +// - username string func (_e *IMetaTable_Expecter) GetCredential(username interface{}) *IMetaTable_GetCredential_Call { return &IMetaTable_GetCredential_Call{Call: _e.mock.On("GetCredential", username)} } @@ -1130,9 +1130,9 @@ type IMetaTable_GetDatabaseByID_Call struct { } // GetDatabaseByID is a helper method to define mock.On call -// - ctx context.Context -// - dbID int64 -// - ts uint64 +// - ctx context.Context +// - dbID int64 +// - ts uint64 func (_e *IMetaTable_Expecter) GetDatabaseByID(ctx interface{}, dbID interface{}, ts interface{}) *IMetaTable_GetDatabaseByID_Call { return &IMetaTable_GetDatabaseByID_Call{Call: _e.mock.On("GetDatabaseByID", ctx, dbID, ts)} } @@ -1186,9 +1186,9 @@ type IMetaTable_GetDatabaseByName_Call struct { } // GetDatabaseByName is a helper method to define mock.On call -// - ctx context.Context -// - dbName string -// - ts uint64 +// - ctx context.Context +// - dbName string +// - ts uint64 func (_e *IMetaTable_Expecter) GetDatabaseByName(ctx interface{}, dbName interface{}, ts interface{}) *IMetaTable_GetDatabaseByName_Call { return &IMetaTable_GetDatabaseByName_Call{Call: _e.mock.On("GetDatabaseByName", ctx, dbName, ts)} } @@ -1232,7 +1232,7 @@ type IMetaTable_GetVChannelsByPchannel_Call struct { } // GetVChannelsByPchannel is a helper method to define mock.On call -// - pchannel string +// - pchannel string func (_e *IMetaTable_Expecter) GetVChannelsByPchannel(pchannel interface{}) *IMetaTable_GetVChannelsByPchannel_Call { return &IMetaTable_GetVChannelsByPchannel_Call{Call: _e.mock.On("GetVChannelsByPchannel", pchannel)} } @@ -1274,8 +1274,8 @@ type IMetaTable_IsAlias_Call struct { } // IsAlias is a helper method to define mock.On call -// - db string -// - name string +// - db string +// - name string func (_e *IMetaTable_Expecter) IsAlias(db interface{}, name interface{}) *IMetaTable_IsAlias_Call { return &IMetaTable_IsAlias_Call{Call: _e.mock.On("IsAlias", db, name)} } @@ -1329,10 +1329,10 @@ type IMetaTable_ListAliases_Call struct { } // ListAliases is a helper method to define mock.On call -// - ctx context.Context -// - dbName string -// - collectionName string -// - ts uint64 +// - ctx context.Context +// - dbName string +// - collectionName string +// - ts uint64 func (_e *IMetaTable_Expecter) ListAliases(ctx interface{}, dbName interface{}, collectionName interface{}, ts interface{}) *IMetaTable_ListAliases_Call { return &IMetaTable_ListAliases_Call{Call: _e.mock.On("ListAliases", ctx, dbName, collectionName, ts)} } @@ -1376,7 +1376,7 @@ type IMetaTable_ListAliasesByID_Call struct { } // ListAliasesByID is a helper method to define mock.On call -// - collID int64 +// - collID int64 func (_e *IMetaTable_Expecter) ListAliasesByID(collID interface{}) *IMetaTable_ListAliasesByID_Call { return &IMetaTable_ListAliasesByID_Call{Call: _e.mock.On("ListAliasesByID", collID)} } @@ -1420,7 +1420,7 @@ type IMetaTable_ListAllAvailCollections_Call struct { } // ListAllAvailCollections is a helper method to define mock.On call -// - ctx context.Context +// - ctx context.Context func (_e *IMetaTable_Expecter) ListAllAvailCollections(ctx interface{}) *IMetaTable_ListAllAvailCollections_Call { return &IMetaTable_ListAllAvailCollections_Call{Call: _e.mock.On("ListAllAvailCollections", ctx)} } @@ -1517,10 +1517,10 @@ type IMetaTable_ListCollections_Call struct { } // ListCollections is a helper method to define mock.On call -// - ctx context.Context -// - dbName string -// - ts uint64 -// - onlyAvail bool +// - ctx context.Context +// - dbName string +// - ts uint64 +// - onlyAvail bool func (_e *IMetaTable_Expecter) ListCollections(ctx interface{}, dbName interface{}, ts interface{}, onlyAvail interface{}) *IMetaTable_ListCollections_Call { return &IMetaTable_ListCollections_Call{Call: _e.mock.On("ListCollections", ctx, dbName, ts, onlyAvail)} } @@ -1627,8 +1627,8 @@ type IMetaTable_ListDatabases_Call struct { } // ListDatabases is a helper method to define mock.On call -// - ctx context.Context -// - ts uint64 +// - ctx context.Context +// - ts uint64 func (_e *IMetaTable_Expecter) ListDatabases(ctx interface{}, ts interface{}) *IMetaTable_ListDatabases_Call { return &IMetaTable_ListDatabases_Call{Call: _e.mock.On("ListDatabases", ctx, ts)} } @@ -1682,7 +1682,7 @@ type IMetaTable_ListPolicy_Call struct { } // ListPolicy is a helper method to define mock.On call -// - tenant string +// - tenant string func (_e *IMetaTable_Expecter) ListPolicy(tenant interface{}) *IMetaTable_ListPolicy_Call { return &IMetaTable_ListPolicy_Call{Call: _e.mock.On("ListPolicy", tenant)} } @@ -1736,7 +1736,7 @@ type IMetaTable_ListUserRole_Call struct { } // ListUserRole is a helper method to define mock.On call -// - tenant string +// - tenant string func (_e *IMetaTable_Expecter) ListUserRole(tenant interface{}) *IMetaTable_ListUserRole_Call { return &IMetaTable_ListUserRole_Call{Call: _e.mock.On("ListUserRole", tenant)} } @@ -1778,9 +1778,9 @@ type IMetaTable_OperatePrivilege_Call struct { } // OperatePrivilege is a helper method to define mock.On call -// - tenant string -// - entity *milvuspb.GrantEntity -// - operateType milvuspb.OperatePrivilegeType +// - tenant string +// - entity *milvuspb.GrantEntity +// - operateType milvuspb.OperatePrivilegeType func (_e *IMetaTable_Expecter) OperatePrivilege(tenant interface{}, entity interface{}, operateType interface{}) *IMetaTable_OperatePrivilege_Call { return &IMetaTable_OperatePrivilege_Call{Call: _e.mock.On("OperatePrivilege", tenant, entity, operateType)} } @@ -1822,10 +1822,10 @@ type IMetaTable_OperateUserRole_Call struct { } // OperateUserRole is a helper method to define mock.On call -// - tenant string -// - userEntity *milvuspb.UserEntity -// - roleEntity *milvuspb.RoleEntity -// - operateType milvuspb.OperateUserRoleType +// - tenant string +// - userEntity *milvuspb.UserEntity +// - roleEntity *milvuspb.RoleEntity +// - operateType milvuspb.OperateUserRoleType func (_e *IMetaTable_Expecter) OperateUserRole(tenant interface{}, userEntity interface{}, roleEntity interface{}, operateType interface{}) *IMetaTable_OperateUserRole_Call { return &IMetaTable_OperateUserRole_Call{Call: _e.mock.On("OperateUserRole", tenant, userEntity, roleEntity, operateType)} } @@ -1867,9 +1867,9 @@ type IMetaTable_RemoveCollection_Call struct { } // RemoveCollection is a helper method to define mock.On call -// - ctx context.Context -// - collectionID int64 -// - ts uint64 +// - ctx context.Context +// - collectionID int64 +// - ts uint64 func (_e *IMetaTable_Expecter) RemoveCollection(ctx interface{}, collectionID interface{}, ts interface{}) *IMetaTable_RemoveCollection_Call { return &IMetaTable_RemoveCollection_Call{Call: _e.mock.On("RemoveCollection", ctx, collectionID, ts)} } @@ -1911,11 +1911,11 @@ type IMetaTable_RemovePartition_Call struct { } // RemovePartition is a helper method to define mock.On call -// - ctx context.Context -// - dbID int64 -// - collectionID int64 -// - partitionID int64 -// - ts uint64 +// - ctx context.Context +// - dbID int64 +// - collectionID int64 +// - partitionID int64 +// - ts uint64 func (_e *IMetaTable_Expecter) RemovePartition(ctx interface{}, dbID interface{}, collectionID interface{}, partitionID interface{}, ts interface{}) *IMetaTable_RemovePartition_Call { return &IMetaTable_RemovePartition_Call{Call: _e.mock.On("RemovePartition", ctx, dbID, collectionID, partitionID, ts)} } @@ -1957,12 +1957,12 @@ type IMetaTable_RenameCollection_Call struct { } // RenameCollection is a helper method to define mock.On call -// - ctx context.Context -// - dbName string -// - oldName string -// - newDBName string -// - newName string -// - ts uint64 +// - ctx context.Context +// - dbName string +// - oldName string +// - newDBName string +// - newName string +// - ts uint64 func (_e *IMetaTable_Expecter) RenameCollection(ctx interface{}, dbName interface{}, oldName interface{}, newDBName interface{}, newName interface{}, ts interface{}) *IMetaTable_RenameCollection_Call { return &IMetaTable_RenameCollection_Call{Call: _e.mock.On("RenameCollection", ctx, dbName, oldName, newDBName, newName, ts)} } @@ -2016,8 +2016,8 @@ type IMetaTable_SelectGrant_Call struct { } // SelectGrant is a helper method to define mock.On call -// - tenant string -// - entity *milvuspb.GrantEntity +// - tenant string +// - entity *milvuspb.GrantEntity func (_e *IMetaTable_Expecter) SelectGrant(tenant interface{}, entity interface{}) *IMetaTable_SelectGrant_Call { return &IMetaTable_SelectGrant_Call{Call: _e.mock.On("SelectGrant", tenant, entity)} } @@ -2071,9 +2071,9 @@ type IMetaTable_SelectRole_Call struct { } // SelectRole is a helper method to define mock.On call -// - tenant string -// - entity *milvuspb.RoleEntity -// - includeUserInfo bool +// - tenant string +// - entity *milvuspb.RoleEntity +// - includeUserInfo bool func (_e *IMetaTable_Expecter) SelectRole(tenant interface{}, entity interface{}, includeUserInfo interface{}) *IMetaTable_SelectRole_Call { return &IMetaTable_SelectRole_Call{Call: _e.mock.On("SelectRole", tenant, entity, includeUserInfo)} } @@ -2127,9 +2127,9 @@ type IMetaTable_SelectUser_Call struct { } // SelectUser is a helper method to define mock.On call -// - tenant string -// - entity *milvuspb.UserEntity -// - includeRoleInfo bool +// - tenant string +// - entity *milvuspb.UserEntity +// - includeRoleInfo bool func (_e *IMetaTable_Expecter) SelectUser(tenant interface{}, entity interface{}, includeRoleInfo interface{}) *IMetaTable_SelectUser_Call { return &IMetaTable_SelectUser_Call{Call: _e.mock.On("SelectUser", tenant, entity, includeRoleInfo)} } diff --git a/internal/streamingnode/client/handler/producer/producer_test.go b/internal/streamingnode/client/handler/producer/producer_test.go index db0092c5be..59cebfd23b 100644 --- a/internal/streamingnode/client/handler/producer/producer_test.go +++ b/internal/streamingnode/client/handler/producer/producer_test.go @@ -59,16 +59,13 @@ func TestProducer(t *testing.T) { assert.NotNil(t, producer) ch := make(chan struct{}) go func() { - msgID, err := producer.Produce(ctx, message.NewMutableMessageBuilder(). - WithMessageType(message.MessageTypeUnknown). - WithPayload([]byte{}). - BuildMutable()) + msg := message.CreateTestEmptyInsertMesage(1, nil) + msgID, err := producer.Produce(ctx, msg) assert.Error(t, err) assert.Nil(t, msgID) - msgID, err = producer.Produce(ctx, message.NewMutableMessageBuilder(). - WithMessageType(message.MessageTypeUnknown). - WithPayload([]byte{}). - BuildMutable()) + + msg = message.CreateTestEmptyInsertMesage(1, nil) + msgID, err = producer.Produce(ctx, msg) assert.NoError(t, err) assert.NotNil(t, msgID) close(ch) @@ -101,10 +98,8 @@ func TestProducer(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, 100*time.Millisecond) defer cancel() - _, err = producer.Produce(ctx, message.NewMutableMessageBuilder(). - WithMessageType(message.MessageTypeUnknown). - WithPayload([]byte{}). - BuildMutable()) + msg := message.CreateTestEmptyInsertMesage(1, nil) + _, err = producer.Produce(ctx, msg) assert.ErrorIs(t, err, context.DeadlineExceeded) assert.True(t, producer.IsAvailable()) producer.Close() diff --git a/internal/streamingnode/server/service/handler/producer/produce_server.go b/internal/streamingnode/server/service/handler/producer/produce_server.go index 0841f15774..e23cf45f68 100644 --- a/internal/streamingnode/server/service/handler/producer/produce_server.go +++ b/internal/streamingnode/server/service/handler/producer/produce_server.go @@ -143,11 +143,7 @@ func (p *ProduceServer) recvLoop() (err error) { // handleProduce handles the produce message request. func (p *ProduceServer) handleProduce(req *streamingpb.ProduceMessageRequest) { p.logger.Debug("recv produce message from client", zap.Int64("requestID", req.RequestId)) - msg := message.NewMutableMessageBuilder(). - WithPayload(req.GetMessage().GetPayload()). - WithProperties(req.GetMessage().GetProperties()). - BuildMutable() - + msg := message.NewMutableMessage(req.GetMessage().GetPayload(), req.GetMessage().GetProperties()) if err := p.validateMessage(msg); err != nil { p.logger.Warn("produce message validation failed", zap.Int64("requestID", req.RequestId), zap.Error(err)) p.sendProduceResult(req.RequestId, nil, err) diff --git a/internal/streamingnode/server/wal/adaptor/message_handler.go b/internal/streamingnode/server/wal/adaptor/message_handler.go new file mode 100644 index 0000000000..7dfc7aa6f0 --- /dev/null +++ b/internal/streamingnode/server/wal/adaptor/message_handler.go @@ -0,0 +1,131 @@ +package adaptor + +import ( + "context" + + "go.uber.org/zap" + + "github.com/milvus-io/milvus/internal/streamingnode/server/wal" + "github.com/milvus-io/milvus/pkg/log" + "github.com/milvus-io/milvus/pkg/mq/msgstream" + "github.com/milvus-io/milvus/pkg/streaming/util/message" + "github.com/milvus-io/milvus/pkg/streaming/util/message/adaptor" + "github.com/milvus-io/milvus/pkg/util/typeutil" +) + +type defaultMessageHandler chan message.ImmutableMessage + +func (h defaultMessageHandler) Handle(ctx context.Context, upstream <-chan message.ImmutableMessage, msg message.ImmutableMessage) (incoming message.ImmutableMessage, ok bool, err error) { + var sendingCh chan message.ImmutableMessage + if msg != nil { + sendingCh = h + } + select { + case <-ctx.Done(): + return nil, false, ctx.Err() + case msg, ok := <-upstream: + if !ok { + return nil, false, wal.ErrUpstreamClosed + } + return msg, false, nil + case sendingCh <- msg: + return nil, true, nil + } +} + +func (d defaultMessageHandler) Close() { + close(d) +} + +// NewMsgPackAdaptorHandler create a new message pack adaptor handler. +func NewMsgPackAdaptorHandler() *MsgPackAdaptorHandler { + return &MsgPackAdaptorHandler{ + logger: log.With(), + channel: make(chan *msgstream.MsgPack), + pendings: make([]message.ImmutableMessage, 0), + pendingMsgPack: typeutil.NewMultipartQueue[*msgstream.MsgPack](), + } +} + +// MsgPackAdaptorHandler is the handler for message pack. +type MsgPackAdaptorHandler struct { + logger *log.MLogger + channel chan *msgstream.MsgPack + pendings []message.ImmutableMessage // pendings hold the vOld message which has same time tick. + pendingMsgPack *typeutil.MultipartQueue[*msgstream.MsgPack] // pendingMsgPack hold unsent msgPack. +} + +// Chan is the channel for message. +func (m *MsgPackAdaptorHandler) Chan() <-chan *msgstream.MsgPack { + return m.channel +} + +// Handle is the callback for handling message. +func (m *MsgPackAdaptorHandler) Handle(ctx context.Context, upstream <-chan message.ImmutableMessage, msg message.ImmutableMessage) (incoming message.ImmutableMessage, ok bool, err error) { + // not handle new message if there are pending msgPack. + if msg != nil && m.pendingMsgPack.Len() == 0 { + m.generateMsgPack(msg) + ok = true + } + + for { + var sendCh chan<- *msgstream.MsgPack + if m.pendingMsgPack.Len() != 0 { + sendCh = m.channel + } + + select { + case <-ctx.Done(): + return nil, ok, ctx.Err() + case msg, notClose := <-upstream: + if !notClose { + return nil, ok, wal.ErrUpstreamClosed + } + return msg, ok, nil + case sendCh <- m.pendingMsgPack.Next(): + m.pendingMsgPack.UnsafeAdvance() + if m.pendingMsgPack.Len() > 0 { + continue + } + return nil, ok, nil + } + } +} + +// generateMsgPack generate msgPack from message. +func (m *MsgPackAdaptorHandler) generateMsgPack(msg message.ImmutableMessage) { + switch msg.Version() { + case message.VersionOld: + if len(m.pendings) != 0 { + if msg.TimeTick() > m.pendings[0].TimeTick() { + m.addMsgPackIntoPending(m.pendings...) + m.pendings = nil + } + } + m.pendings = append(m.pendings, msg) + case message.VersionV1: + if len(m.pendings) != 0 { // all previous message should be vOld. + m.addMsgPackIntoPending(m.pendings...) + m.pendings = nil + } + m.addMsgPackIntoPending(msg) + default: + panic("unsupported message version") + } +} + +// addMsgPackIntoPending add message into pending msgPack. +func (m *MsgPackAdaptorHandler) addMsgPackIntoPending(msgs ...message.ImmutableMessage) { + newPack, err := adaptor.NewMsgPackFromMessage(msgs...) + if err != nil { + m.logger.Warn("failed to convert message to msgpack", zap.Error(err)) + } + if newPack != nil { + m.pendingMsgPack.AddOne(newPack) + } +} + +// Close close the handler. +func (m *MsgPackAdaptorHandler) Close() { + close(m.channel) +} diff --git a/internal/streamingnode/server/wal/adaptor/message_handler_test.go b/internal/streamingnode/server/wal/adaptor/message_handler_test.go new file mode 100644 index 0000000000..c84fb225b8 --- /dev/null +++ b/internal/streamingnode/server/wal/adaptor/message_handler_test.go @@ -0,0 +1,76 @@ +package adaptor + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/milvus-io/milvus/pkg/mocks/streaming/util/mock_message" + "github.com/milvus-io/milvus/pkg/streaming/util/message" + "github.com/milvus-io/milvus/pkg/streaming/walimpls/impls/rmq" +) + +func TestMsgPackAdaptorHandler(t *testing.T) { + messageID := rmq.NewRmqID(1) + tt := uint64(100) + msg := message.CreateTestInsertMessage( + t, + 1, + 1000, + tt, + messageID, + ) + immutableMsg := msg.IntoImmutableMessage(messageID) + + upstream := make(chan message.ImmutableMessage, 1) + + ctx := context.Background() + h := NewMsgPackAdaptorHandler() + done := make(chan struct{}) + go func() { + for range h.Chan() { + } + close(done) + }() + upstream <- immutableMsg + newMsg, ok, err := h.Handle(ctx, upstream, nil) + assert.Equal(t, newMsg, immutableMsg) + assert.False(t, ok) + assert.NoError(t, err) + + newMsg, ok, err = h.Handle(ctx, upstream, newMsg) + assert.NoError(t, err) + assert.Nil(t, newMsg) + assert.True(t, ok) + h.Close() + + <-done +} + +func TestDefaultHandler(t *testing.T) { + h := make(defaultMessageHandler, 1) + done := make(chan struct{}) + go func() { + for range h { + } + close(done) + }() + + upstream := make(chan message.ImmutableMessage, 1) + msg := mock_message.NewMockImmutableMessage(t) + upstream <- msg + newMsg, ok, err := h.Handle(context.Background(), upstream, nil) + assert.NotNil(t, newMsg) + assert.NoError(t, err) + assert.False(t, ok) + assert.Equal(t, newMsg, msg) + + newMsg, ok, err = h.Handle(context.Background(), upstream, newMsg) + assert.NoError(t, err) + assert.Nil(t, newMsg) + assert.True(t, ok) + + h.Close() + <-done +} diff --git a/internal/streamingnode/server/wal/adaptor/scanner_adaptor.go b/internal/streamingnode/server/wal/adaptor/scanner_adaptor.go index 9861fb680b..f48b6f6b7b 100644 --- a/internal/streamingnode/server/wal/adaptor/scanner_adaptor.go +++ b/internal/streamingnode/server/wal/adaptor/scanner_adaptor.go @@ -10,6 +10,7 @@ import ( "github.com/milvus-io/milvus/pkg/streaming/util/types" "github.com/milvus-io/milvus/pkg/streaming/walimpls" "github.com/milvus-io/milvus/pkg/streaming/walimpls/helper" + "github.com/milvus-io/milvus/pkg/util/typeutil" ) var _ wal.Scanner = (*scannerAdaptorImpl)(nil) @@ -21,13 +22,15 @@ func newScannerAdaptor( readOption wal.ReadOption, cleanup func(), ) wal.Scanner { + if readOption.MesasgeHandler == nil { + readOption.MesasgeHandler = defaultMessageHandler(make(chan message.ImmutableMessage)) + } s := &scannerAdaptorImpl{ logger: log.With(zap.String("name", name), zap.String("channel", l.Channel().Name)), innerWAL: l, readOption: readOption, - sendingCh: make(chan message.ImmutableMessage, 1), reorderBuffer: utility.NewReOrderBuffer(), - pendingQueue: utility.NewImmutableMessageQueue(), + pendingQueue: typeutil.NewMultipartQueue[message.ImmutableMessage](), cleanup: cleanup, ScannerHelper: helper.NewScannerHelper(name), } @@ -41,9 +44,8 @@ type scannerAdaptorImpl struct { logger *log.MLogger innerWAL walimpls.WALImpls readOption wal.ReadOption - sendingCh chan message.ImmutableMessage - reorderBuffer *utility.ReOrderByTimeTickBuffer // only support time tick reorder now. - pendingQueue *utility.ImmutableMessageQueue // + reorderBuffer *utility.ReOrderByTimeTickBuffer // only support time tick reorder now. + pendingQueue *typeutil.MultipartQueue[message.ImmutableMessage] // cleanup func() } @@ -52,9 +54,9 @@ func (s *scannerAdaptorImpl) Channel() types.PChannelInfo { return s.innerWAL.Channel() } -// Chan returns the channel of message. +// Chan returns the message channel of the scanner. func (s *scannerAdaptorImpl) Chan() <-chan message.ImmutableMessage { - return s.sendingCh + return s.readOption.MesasgeHandler.(defaultMessageHandler) } // Close the scanner, release the underlying resources. @@ -68,7 +70,7 @@ func (s *scannerAdaptorImpl) Close() error { } func (s *scannerAdaptorImpl) executeConsume() { - defer close(s.sendingCh) + defer s.readOption.MesasgeHandler.Close() innerScanner, err := s.innerWAL.Read(s.Context(), walimpls.ReadOption{ Name: s.Name(), @@ -83,36 +85,29 @@ func (s *scannerAdaptorImpl) executeConsume() { for { // generate the event channel and do the event loop. // TODO: Consume from local cache. - upstream, sending := s.getEventCh(innerScanner) - select { - case <-s.Context().Done(): + upstream := s.getUpstream(innerScanner) + + msg, ok, err := s.readOption.MesasgeHandler.Handle(s.Context(), upstream, s.pendingQueue.Next()) + if err != nil { s.Finish(err) return - case msg, ok := <-upstream: - if !ok { - s.Finish(innerScanner.Error()) - return - } - s.handleUpstream(msg) - case sending <- s.pendingQueue.Next(): + } + if ok { s.pendingQueue.UnsafeAdvance() } + if msg != nil { + s.handleUpstream(msg) + } } } -func (s *scannerAdaptorImpl) getEventCh(scanner walimpls.ScannerImpls) (<-chan message.ImmutableMessage, chan<- message.ImmutableMessage) { - if s.pendingQueue.Len() == 0 { - // If pending queue is empty, - // no more message can be sent, - // we always need to recv message from upstream to avoid starve. - return scanner.Chan(), nil - } +func (s *scannerAdaptorImpl) getUpstream(scanner walimpls.ScannerImpls) <-chan message.ImmutableMessage { // TODO: configurable pending buffer count. // If the pending queue is full, we need to wait until it's consumed to avoid scanner overloading. if s.pendingQueue.Len() > 16 { - return nil, s.sendingCh + return nil } - return scanner.Chan(), s.sendingCh + return scanner.Chan() } func (s *scannerAdaptorImpl) handleUpstream(msg message.ImmutableMessage) { diff --git a/internal/streamingnode/server/wal/adaptor/wal_test.go b/internal/streamingnode/server/wal/adaptor/wal_test.go index aad5964f5a..def37b7c32 100644 --- a/internal/streamingnode/server/wal/adaptor/wal_test.go +++ b/internal/streamingnode/server/wal/adaptor/wal_test.go @@ -10,11 +10,9 @@ import ( "testing" "time" - "github.com/golang/protobuf/proto" "github.com/remeh/sizedwaitgroup" "github.com/stretchr/testify/assert" - "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus/internal/streamingnode/server/resource" "github.com/milvus-io/milvus/internal/streamingnode/server/resource/idalloc" "github.com/milvus-io/milvus/internal/streamingnode/server/wal" @@ -154,26 +152,10 @@ func (f *testOneWALFramework) testAppend(ctx context.Context, w wal.WAL) ([]mess time.Sleep(time.Duration(5+rand.Int31n(10)) * time.Millisecond) // ...rocksmq has a dirty implement of properties, // without commonpb.MsgHeader, it can not work. - header := commonpb.MsgHeader{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_Insert, - MsgID: int64(i), - }, - } - payload, err := proto.Marshal(&header) - if err != nil { - panic(err) - } - properties := map[string]string{ + msg := message.CreateTestEmptyInsertMesage(int64(i), map[string]string{ "id": fmt.Sprintf("%d", i), "const": "t", - } - typ := message.MessageTypeUnknown - msg := message.NewMutableMessageBuilder(). - WithMessageType(typ). - WithPayload(payload). - WithProperties(properties). - BuildMutable() + }) id, err := w.Append(ctx, msg) assert.NoError(f.t, err) assert.NotNil(f.t, id) @@ -181,27 +163,12 @@ func (f *testOneWALFramework) testAppend(ctx context.Context, w wal.WAL) ([]mess }(i) } swg.Wait() - // send a final hint message - header := commonpb.MsgHeader{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_Insert, - MsgID: int64(f.messageCount - 1), - }, - } - payload, err := proto.Marshal(&header) - if err != nil { - panic(err) - } - properties := map[string]string{ + + msg := message.CreateTestEmptyInsertMesage(int64(f.messageCount-1), map[string]string{ "id": fmt.Sprintf("%d", f.messageCount-1), "const": "t", "term": strconv.FormatInt(int64(f.term), 10), - } - msg := message.NewMutableMessageBuilder(). - WithPayload(payload). - WithProperties(properties). - WithMessageType(message.MessageTypeUnknown). - BuildMutable() + }) id, err := w.Append(ctx, msg) assert.NoError(f.t, err) messages[f.messageCount-1] = msg.IntoImmutableMessage(id) diff --git a/internal/streamingnode/server/wal/interceptors/timetick/timetick_message.go b/internal/streamingnode/server/wal/interceptors/timetick/timetick_message.go index fc2f52df4e..e654a521aa 100644 --- a/internal/streamingnode/server/wal/interceptors/timetick/timetick_message.go +++ b/internal/streamingnode/server/wal/interceptors/timetick/timetick_message.go @@ -1,48 +1,28 @@ package timetick import ( - "github.com/cockroachdb/errors" - "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" - "github.com/milvus-io/milvus/pkg/mq/msgstream" "github.com/milvus-io/milvus/pkg/streaming/util/message" "github.com/milvus-io/milvus/pkg/util/commonpbutil" ) func newTimeTickMsg(ts uint64, sourceID int64) (message.MutableMessage, error) { // TODO: time tick should be put on properties, for compatibility, we put it on message body now. - msgstreamMsg := &msgstream.TimeTickMsg{ - BaseMsg: msgstream.BaseMsg{ - BeginTimestamp: ts, - EndTimestamp: ts, - HashValues: []uint32{0}, - }, - TimeTickMsg: msgpb.TimeTickMsg{ + // Common message's time tick is set on interceptor. + // TimeTickMsg's time tick should be set here. + msg, err := message.NewTimeTickMessageBuilderV1(). + WithMessageHeader(&message.TimeTickMessageHeader{}). + WithPayload(&msgpb.TimeTickMsg{ Base: commonpbutil.NewMsgBase( commonpbutil.WithMsgType(commonpb.MsgType_TimeTick), commonpbutil.WithMsgID(0), commonpbutil.WithTimeStamp(ts), commonpbutil.WithSourceID(sourceID), ), - }, - } - bytes, err := msgstreamMsg.Marshal(msgstreamMsg) + }).BuildMutable() if err != nil { - return nil, errors.Wrap(err, "marshal time tick message failed") + return nil, err } - - payload, ok := bytes.([]byte) - if !ok { - return nil, errors.New("marshal time tick message as []byte failed") - } - - // Common message's time tick is set on interceptor. - // TimeTickMsg's time tick should be set here. - msg := message.NewMutableMessageBuilder(). - WithMessageType(message.MessageTypeTimeTick). - WithPayload(payload). - BuildMutable(). - WithTimeTick(ts) - return msg, nil + return msg.WithTimeTick(ts), nil } diff --git a/internal/streamingnode/server/wal/scanner.go b/internal/streamingnode/server/wal/scanner.go index f9ea7a65a2..8a5ed86464 100644 --- a/internal/streamingnode/server/wal/scanner.go +++ b/internal/streamingnode/server/wal/scanner.go @@ -1,6 +1,10 @@ package wal import ( + "context" + + "github.com/cockroachdb/errors" + "github.com/milvus-io/milvus/pkg/streaming/util/message" "github.com/milvus-io/milvus/pkg/streaming/util/options" "github.com/milvus-io/milvus/pkg/streaming/util/types" @@ -8,15 +12,22 @@ import ( type MessageFilter = func(message.ImmutableMessage) bool +var ErrUpstreamClosed = errors.New("upstream closed") + // ReadOption is the option for reading records from the wal. type ReadOption struct { - DeliverPolicy options.DeliverPolicy - MessageFilter MessageFilter + DeliverPolicy options.DeliverPolicy + MessageFilter MessageFilter + MesasgeHandler MessageHandler // message handler for message processing. + // If the message handler is nil (no redundant operation need to apply), + // the default message handler will be used, and the receiver will be returned from Chan. + // Otherwise, Chan will panic. + // vaild every message will be passed to this handler before being delivered to the consumer. } // Scanner is the interface for reading records from the wal. type Scanner interface { - // Chan returns the channel of message. + // Chan returns the channel of message if Option.MessageHandler is nil. Chan() <-chan message.ImmutableMessage // Channel returns the channel assignment info of the wal. @@ -33,3 +44,19 @@ type Scanner interface { // Return the error same with `Error` Close() error } + +// MessageHandler is used to handle message read from log. +// TODO: should be removed in future after msgstream is removed. +type MessageHandler interface { + // Handle is the callback for handling message. + // The message will be passed to the handler for processing. + // Handle operation can be blocked, but should listen to the context.Done() and upstream. + // If the context is canceled, the handler should return immediately with ctx.Err. + // If the upstream is closed, the handler should return immediately with ErrUpstreamClosed. + // If the upstream recv a message, the handler should return the incoming message. + // If the handler handle the message successfully, it should return the ok=true. + Handle(ctx context.Context, upstream <-chan message.ImmutableMessage, msg message.ImmutableMessage) (incoming message.ImmutableMessage, ok bool, err error) + + // Close is called after all messages are handled or handling is interrupted. + Close() +} diff --git a/internal/streamingnode/server/wal/utility/immutable_message_queue.go b/internal/streamingnode/server/wal/utility/immutable_message_queue.go deleted file mode 100644 index 75e72bc8a0..0000000000 --- a/internal/streamingnode/server/wal/utility/immutable_message_queue.go +++ /dev/null @@ -1,51 +0,0 @@ -package utility - -import "github.com/milvus-io/milvus/pkg/streaming/util/message" - -// NewImmutableMessageQueue create a new immutable message queue. -func NewImmutableMessageQueue() *ImmutableMessageQueue { - return &ImmutableMessageQueue{ - pendings: make([][]message.ImmutableMessage, 0), - cnt: 0, - } -} - -// ImmutableMessageQueue is a queue of messages. -type ImmutableMessageQueue struct { - pendings [][]message.ImmutableMessage - cnt int -} - -// Len return the queue size. -func (pq *ImmutableMessageQueue) Len() int { - return pq.cnt -} - -// Add add a slice of message as pending one -func (pq *ImmutableMessageQueue) Add(msgs []message.ImmutableMessage) { - if len(msgs) == 0 { - return - } - pq.pendings = append(pq.pendings, msgs) - pq.cnt += len(msgs) -} - -// Next return the next message in pending queue. -func (pq *ImmutableMessageQueue) Next() message.ImmutableMessage { - if len(pq.pendings) != 0 && len(pq.pendings[0]) != 0 { - return pq.pendings[0][0] - } - return nil -} - -// UnsafeAdvance do a advance without check. -// !!! Should only be called `Next` do not return nil. -func (pq *ImmutableMessageQueue) UnsafeAdvance() { - if len(pq.pendings[0]) == 1 { - pq.pendings = pq.pendings[1:] - pq.cnt-- - return - } - pq.pendings[0] = pq.pendings[0][1:] - pq.cnt-- -} diff --git a/internal/streamingnode/server/wal/utility/immutable_message_queue_test.go b/internal/streamingnode/server/wal/utility/immutable_message_queue_test.go deleted file mode 100644 index 270b048b15..0000000000 --- a/internal/streamingnode/server/wal/utility/immutable_message_queue_test.go +++ /dev/null @@ -1,25 +0,0 @@ -package utility - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/milvus-io/milvus/pkg/mocks/streaming/util/mock_message" - "github.com/milvus-io/milvus/pkg/streaming/util/message" -) - -func TestImmutableMessageQueue(t *testing.T) { - q := NewImmutableMessageQueue() - for i := 0; i < 100; i++ { - q.Add([]message.ImmutableMessage{ - mock_message.NewMockImmutableMessage(t), - }) - assert.Equal(t, i+1, q.Len()) - } - for i := 100; i > 0; i-- { - assert.NotNil(t, q.Next()) - q.UnsafeAdvance() - assert.Equal(t, i-1, q.Len()) - } -} diff --git a/internal/util/flowgraph/message_test.go b/internal/util/flowgraph/message_test.go index 720aa631f2..46125f9812 100644 --- a/internal/util/flowgraph/message_test.go +++ b/internal/util/flowgraph/message_test.go @@ -45,6 +45,9 @@ func (bm *MockMsg) SetID(id msgstream.UniqueID) { // do nothing } +func (bm *MockMsg) SetTs(ts uint64) { +} + func (bm *MockMsg) BeginTs() Timestamp { return 0 } diff --git a/pkg/Makefile b/pkg/Makefile index cb09dd830d..595d73bc32 100644 --- a/pkg/Makefile +++ b/pkg/Makefile @@ -17,4 +17,7 @@ generate-mockery: getdeps $(INSTALL_PATH)/mockery --name=Factory --dir=$(PWD)/mq/msgstream --output=$(PWD)/mq/msgstream --filename=mock_msgstream_factory.go --with-expecter --structname=MockFactory --outpkg=msgstream --inpackage $(INSTALL_PATH)/mockery --name=Client --dir=$(PWD)/mq/msgdispatcher --output=$(PWD)/mq/msgsdispatcher --filename=mock_client.go --with-expecter --structname=MockClient --outpkg=msgdispatcher --inpackage $(INSTALL_PATH)/mockery --name=Logger --dir=$(PWD)/eventlog --output=$(PWD)/eventlog --filename=mock_logger.go --with-expecter --structname=MockLogger --outpkg=eventlog --inpackage - $(INSTALL_PATH)/mockery --name=MessageID --dir=$(PWD)/mq/msgstream/mqwrapper --output=$(PWD)/mq/msgstream/mqwrapper --filename=mock_id.go --with-expecter --structname=MockMessageID --outpkg=mqwrapper --inpackage \ No newline at end of file + $(INSTALL_PATH)/mockery --name=MessageID --dir=$(PWD)/mq/msgstream/mqwrapper --output=$(PWD)/mq/msgstream/mqwrapper --filename=mock_id.go --with-expecter --structname=MockMessageID --outpkg=mqwrapper --inpackage + +generate-proto: + $(ROOTPATH)/cmake_build/bin/protoc --proto_path=$(PWD)/streaming/util/message/messagepb --go_out=plugins=grpc,paths=source_relative:./streaming/util/message/messagepb $(PWD)/streaming/util/message/messagepb/message.proto \ No newline at end of file diff --git a/pkg/mq/msgstream/mq_msgstream.go b/pkg/mq/msgstream/mq_msgstream.go index ccf1d6bd86..e322a7f279 100644 --- a/pkg/mq/msgstream/mq_msgstream.go +++ b/pkg/mq/msgstream/mq_msgstream.go @@ -384,6 +384,11 @@ func (ms *mqMsgStream) Broadcast(msgPack *MsgPack) (map[string][]MessageID, erro } func (ms *mqMsgStream) getTsMsgFromConsumerMsg(msg common.Message) (TsMsg, error) { + return GetTsMsgFromConsumerMsg(ms.unmarshal, msg) +} + +// GetTsMsgFromConsumerMsg get TsMsg from consumer message +func GetTsMsgFromConsumerMsg(unmarshalDispatcher UnmarshalDispatcher, msg common.Message) (TsMsg, error) { header := commonpb.MsgHeader{} if msg.Payload() == nil { return nil, fmt.Errorf("failed to unmarshal message header, payload is empty") @@ -395,7 +400,7 @@ func (ms *mqMsgStream) getTsMsgFromConsumerMsg(msg common.Message) (TsMsg, error if header.Base == nil { return nil, fmt.Errorf("failed to unmarshal message, header is uncomplete") } - tsMsg, err := ms.unmarshal.Unmarshal(msg.Payload(), header.Base.MsgType) + tsMsg, err := unmarshalDispatcher.Unmarshal(msg.Payload(), header.Base.MsgType) if err != nil { return nil, fmt.Errorf("failed to unmarshal tsMsg, err %s", err.Error()) } diff --git a/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_id.go b/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_id.go index f534e17da5..e029175a63 100644 --- a/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_id.go +++ b/pkg/mq/msgstream/mqwrapper/pulsar/pulsar_id.go @@ -24,6 +24,13 @@ import ( "github.com/milvus-io/milvus/pkg/mq/common" ) +// NewPulsarID creates a new pulsarID +func NewPulsarID(id pulsar.MessageID) *pulsarID { + return &pulsarID{ + messageID: id, + } +} + type pulsarID struct { messageID pulsar.MessageID } @@ -31,6 +38,10 @@ type pulsarID struct { // Check if pulsarID implements and MessageID interface var _ common.MessageID = &pulsarID{} +func (pid *pulsarID) PulsarID() pulsar.MessageID { + return pid.messageID +} + func (pid *pulsarID) Serialize() []byte { return pid.messageID.Serialize() } diff --git a/pkg/mq/msgstream/msg.go b/pkg/mq/msgstream/msg.go index 2db757a0f6..0ae1affbee 100644 --- a/pkg/mq/msgstream/msg.go +++ b/pkg/mq/msgstream/msg.go @@ -53,6 +53,7 @@ type TsMsg interface { Unmarshal(MarshalType) (TsMsg, error) Position() *MsgPosition SetPosition(*MsgPosition) + SetTs(ts uint64) Size() int } @@ -111,6 +112,11 @@ func (bm *BaseMsg) SetPosition(position *MsgPosition) { bm.MsgPosition = position } +func (bm *BaseMsg) SetTs(ts uint64) { + bm.BeginTimestamp = ts + bm.EndTimestamp = ts +} + func convertToByteArray(input interface{}) ([]byte, error) { switch output := input.(type) { case []byte: diff --git a/pkg/streaming/OWNERS b/pkg/streaming/OWNERS new file mode 100644 index 0000000000..3895caf6d6 --- /dev/null +++ b/pkg/streaming/OWNERS @@ -0,0 +1,5 @@ +reviewers: + - chyezh + +approvers: + - maintainers \ No newline at end of file diff --git a/pkg/streaming/util/message/adaptor/message.go b/pkg/streaming/util/message/adaptor/message.go new file mode 100644 index 0000000000..e9d012acd1 --- /dev/null +++ b/pkg/streaming/util/message/adaptor/message.go @@ -0,0 +1,124 @@ +package adaptor + +import ( + "github.com/cockroachdb/errors" + + "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" + "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" + "github.com/milvus-io/milvus/pkg/mq/msgstream" + "github.com/milvus-io/milvus/pkg/streaming/util/message" +) + +var unmashalerDispatcher = (&msgstream.ProtoUDFactory{}).NewUnmarshalDispatcher() + +// FromMessageToMsgPack converts message to msgpack. +// Same TimeTick must be sent with same msgpack. +// !!! Msgs must be keep same time tick. +// TODO: remove this function after remove the msgstream implementation. +func NewMsgPackFromMessage(msgs ...message.ImmutableMessage) (*msgstream.MsgPack, error) { + if len(msgs) == 0 { + return nil, nil + } + allTsMsgs := make([]msgstream.TsMsg, 0, len(msgs)) + + var finalErr error + for _, msg := range msgs { + var tsMsg msgstream.TsMsg + var err error + switch msg.Version() { + case message.VersionOld: + tsMsg, err = fromMessageToTsMsgVOld(msg) + case message.VersionV1: + tsMsg, err = fromMessageToTsMsgV1(msg) + default: + panic("unsupported message version") + } + if err != nil { + finalErr = errors.CombineErrors(finalErr, errors.Wrapf(err, "Failed to convert message to msgpack, %v", msg.MessageID())) + continue + } + allTsMsgs = append(allTsMsgs, tsMsg) + } + if len(allTsMsgs) == 0 { + return nil, finalErr + } + + // msgs is sorted by time tick. + // Postition use the last confirmed message id. + // 1. So use the first tsMsgs's Position can read all messages which timetick is greater or equal than the first tsMsgs's BeginTs. + // In other words, from the StartPositions, you can read the full msgPack. + // 2. Use the last tsMsgs's Position as the EndPosition, you can read all messages following the msgPack. + return &msgstream.MsgPack{ + BeginTs: allTsMsgs[0].BeginTs(), + EndTs: allTsMsgs[len(allTsMsgs)-1].EndTs(), + Msgs: allTsMsgs, + StartPositions: []*msgstream.MsgPosition{allTsMsgs[0].Position()}, + EndPositions: []*msgstream.MsgPosition{allTsMsgs[len(allTsMsgs)-1].Position()}, + }, finalErr +} + +func fromMessageToTsMsgVOld(msg message.ImmutableMessage) (msgstream.TsMsg, error) { + panic("Not implemented") +} + +// fromMessageToTsMsgV1 converts message to ts message. +func fromMessageToTsMsgV1(msg message.ImmutableMessage) (msgstream.TsMsg, error) { + tsMsg, err := unmashalerDispatcher.Unmarshal(msg.Payload(), commonpb.MsgType(msg.MessageType())) + if err != nil { + return nil, errors.Wrap(err, "Failed to unmarshal message") + } + tsMsg.SetTs(msg.TimeTick()) + tsMsg.SetPosition(&msgpb.MsgPosition{ + ChannelName: msg.VChannel(), + // from the last confirmed message id, you can read all messages which timetick is greater or equal than current message id. + MsgID: MustGetMQWrapperIDFromMessage(msg.LastConfirmedMessageID()).Serialize(), + MsgGroup: "", // Not important any more. + Timestamp: msg.TimeTick(), + }) + + return recoverMessageFromHeader(tsMsg, msg) +} + +// recoverMessageFromHeader recovers message from header. +func recoverMessageFromHeader(tsMsg msgstream.TsMsg, msg message.ImmutableMessage) (msgstream.TsMsg, error) { + switch msg.MessageType() { + case message.MessageTypeInsert: + insertMessage, err := message.AsImmutableInsertMessage(msg) + if err != nil { + return nil, errors.Wrap(err, "Failed to convert message to insert message") + } + // insertMsg has multiple partition and segment assignment is done by insert message header. + // so recover insert message from header before send it. + return recoverInsertMsgFromHeader(tsMsg.(*msgstream.InsertMsg), insertMessage.MessageHeader(), msg.TimeTick()) + default: + return tsMsg, nil + } +} + +// recoverInsertMsgFromHeader recovers insert message from header. +func recoverInsertMsgFromHeader(insertMsg *msgstream.InsertMsg, header *message.InsertMessageHeader, timetick uint64) (msgstream.TsMsg, error) { + if insertMsg.GetCollectionID() != header.GetCollectionId() { + panic("unreachable code, collection id is not equal") + } + // header promise a batch insert on vchannel in future, so header has multiple partition. + var assignment *message.PartitionSegmentAssignment + for _, p := range header.Partitions { + if p.GetPartitionId() == insertMsg.GetPartitionID() { + assignment = p + break + } + } + if assignment.GetSegmentAssignment().GetSegmentId() == 0 { + panic("unreachable code, partition id is not exist") + } + + insertMsg.SegmentID = assignment.GetSegmentAssignment().GetSegmentId() + // timetick should has been assign at streaming node. + // so overwrite the timetick on insertRequest. + timestamps := make([]uint64, insertMsg.GetNumRows()) + for i := 0; i < len(timestamps); i++ { + timestamps[i] = timetick + } + insertMsg.Timestamps = timestamps + return insertMsg, nil +} diff --git a/pkg/streaming/util/message/adaptor/message_id.go b/pkg/streaming/util/message/adaptor/message_id.go new file mode 100644 index 0000000000..645c9c8b2f --- /dev/null +++ b/pkg/streaming/util/message/adaptor/message_id.go @@ -0,0 +1,34 @@ +package adaptor + +import ( + "github.com/apache/pulsar-client-go/pulsar" + + "github.com/milvus-io/milvus/pkg/mq/common" + "github.com/milvus-io/milvus/pkg/mq/mqimpl/rocksmq/server" + mqpulsar "github.com/milvus-io/milvus/pkg/mq/msgstream/mqwrapper/pulsar" + "github.com/milvus-io/milvus/pkg/streaming/util/message" + msgpulsar "github.com/milvus-io/milvus/pkg/streaming/walimpls/impls/pulsar" + "github.com/milvus-io/milvus/pkg/streaming/walimpls/impls/rmq" +) + +// MustGetMQWrapperIDFromMessage converts message.MessageID to common.MessageID +// TODO: should be removed in future after common.MessageID is removed +func MustGetMQWrapperIDFromMessage(messageID message.MessageID) common.MessageID { + if id, ok := messageID.(interface{ PulsarID() pulsar.MessageID }); ok { + return mqpulsar.NewPulsarID(id.PulsarID()) + } else if id, ok := messageID.(interface{ RmqID() int64 }); ok { + return &server.RmqID{MessageID: id.RmqID()} + } + panic("unsupported now") +} + +// MustGetMessageIDFromMQWrapperID converts common.MessageID to message.MessageID +// TODO: should be removed in future after common.MessageID is removed +func MustGetMessageIDFromMQWrapperID(commonMessageID common.MessageID) message.MessageID { + if id, ok := commonMessageID.(interface{ PulsarID() pulsar.MessageID }); ok { + return msgpulsar.NewPulsarID(id.PulsarID()) + } else if id, ok := commonMessageID.(*server.RmqID); ok { + return rmq.NewRmqID(id.MessageID) + } + return nil +} diff --git a/pkg/streaming/util/message/adaptor/message_id_test.go b/pkg/streaming/util/message/adaptor/message_id_test.go new file mode 100644 index 0000000000..e5216e064b --- /dev/null +++ b/pkg/streaming/util/message/adaptor/message_id_test.go @@ -0,0 +1,20 @@ +package adaptor + +import ( + "testing" + + "github.com/apache/pulsar-client-go/pulsar" + "github.com/stretchr/testify/assert" + + msgpulsar "github.com/milvus-io/milvus/pkg/streaming/walimpls/impls/pulsar" + "github.com/milvus-io/milvus/pkg/streaming/walimpls/impls/rmq" +) + +func TestIDCoversion(t *testing.T) { + id := MustGetMessageIDFromMQWrapperID(MustGetMQWrapperIDFromMessage(rmq.NewRmqID(1))) + assert.True(t, id.EQ(rmq.NewRmqID(1))) + + msgID := pulsar.EarliestMessageID() + id = MustGetMessageIDFromMQWrapperID(MustGetMQWrapperIDFromMessage(msgpulsar.NewPulsarID(msgID))) + assert.True(t, id.EQ(msgpulsar.NewPulsarID(msgID))) +} diff --git a/pkg/streaming/util/message/adaptor/message_test.go b/pkg/streaming/util/message/adaptor/message_test.go new file mode 100644 index 0000000000..76038ddd3f --- /dev/null +++ b/pkg/streaming/util/message/adaptor/message_test.go @@ -0,0 +1,72 @@ +package adaptor + +import ( + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/milvus-io/milvus/pkg/mq/msgstream" + "github.com/milvus-io/milvus/pkg/streaming/util/message" + "github.com/milvus-io/milvus/pkg/streaming/walimpls/impls/rmq" +) + +func TestNewMsgPackFromInsertMessage(t *testing.T) { + id := rmq.NewRmqID(1) + + fieldCount := map[int64]int{ + 3: 1000, + 4: 2000, + 5: 3000, + 6: 5000, + } + tt := uint64(time.Now().UnixNano()) + immutableMessages := make([]message.ImmutableMessage, 0, len(fieldCount)) + for segmentID, rowNum := range fieldCount { + insertMsg := message.CreateTestInsertMessage(t, segmentID, rowNum, tt, id) + immutableMessage := insertMsg.IntoImmutableMessage(id) + immutableMessages = append(immutableMessages, immutableMessage) + } + + pack, err := NewMsgPackFromMessage(immutableMessages...) + assert.NoError(t, err) + assert.NotNil(t, pack) + assert.Equal(t, tt, pack.BeginTs) + assert.Equal(t, tt, pack.EndTs) + assert.Len(t, pack.Msgs, len(fieldCount)) + + for _, msg := range pack.Msgs { + insertMsg := msg.(*msgstream.InsertMsg) + rowNum, ok := fieldCount[insertMsg.GetSegmentID()] + assert.True(t, ok) + + assert.Len(t, insertMsg.Timestamps, rowNum) + assert.Len(t, insertMsg.RowIDs, rowNum) + assert.Len(t, insertMsg.FieldsData, 2) + for _, fieldData := range insertMsg.FieldsData { + if data := fieldData.GetScalars().GetBoolData(); data != nil { + assert.Len(t, data.Data, rowNum) + } else if data := fieldData.GetScalars().GetIntData(); data != nil { + assert.Len(t, data.Data, rowNum) + } + } + + for _, ts := range insertMsg.Timestamps { + assert.Equal(t, ts, tt) + } + } +} + +func TestNewMsgPackFromCreateCollectionMessage(t *testing.T) { + id := rmq.NewRmqID(1) + + tt := uint64(time.Now().UnixNano()) + msg := message.CreateTestCreateCollectionMessage(t, 1, tt, id) + immutableMessage := msg.IntoImmutableMessage(id) + + pack, err := NewMsgPackFromMessage(immutableMessage) + assert.NoError(t, err) + assert.NotNil(t, pack) + assert.Equal(t, tt, pack.BeginTs) + assert.Equal(t, tt, pack.EndTs) +} diff --git a/pkg/streaming/util/message/builder.go b/pkg/streaming/util/message/builder.go index d5a0bfe56f..44abba0604 100644 --- a/pkg/streaming/util/message/builder.go +++ b/pkg/streaming/util/message/builder.go @@ -1,5 +1,24 @@ package message +import ( + "fmt" + "reflect" + + "github.com/cockroachdb/errors" + "github.com/golang/protobuf/proto" + + "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" +) + +// NewMutableMessage creates a new mutable message. +// Only used at server side for streamingnode internal service, don't use it at client side. +func NewMutableMessage(payload []byte, properties map[string]string) MutableMessage { + return &messageImpl{ + payload: payload, + properties: properties, + } +} + // NewImmutableMessage creates a new immutable message. func NewImmutableMesasge( id MessageID, @@ -15,43 +34,69 @@ func NewImmutableMesasge( } } -// NewMutableMessageBuilder creates a new builder. -// Should only used at client side. -func NewMutableMessageBuilder() *MutableMesasgeBuilder { - return &MutableMesasgeBuilder{ - payload: nil, - properties: make(propertiesImpl), +// List all type-safe mutable message builders here. +var ( + NewTimeTickMessageBuilderV1 = createNewMessageBuilderV1[*TimeTickMessageHeader, *msgpb.TimeTickMsg]() + NewInsertMessageBuilderV1 = createNewMessageBuilderV1[*InsertMessageHeader, *msgpb.InsertRequest]() + NewDeleteMessageBuilderV1 = createNewMessageBuilderV1[*DeleteMessageHeader, *msgpb.DeleteRequest]() + NewCreateCollectionMessageBuilderV1 = createNewMessageBuilderV1[*CreateCollectionMessageHeader, *msgpb.CreateCollectionRequest]() + NewDropCollectionMessageBuilderV1 = createNewMessageBuilderV1[*DropCollectionMessageHeader, *msgpb.DropCollectionRequest]() + NewCreatePartitionMessageBuilderV1 = createNewMessageBuilderV1[*CreatePartitionMessageHeader, *msgpb.CreatePartitionRequest]() + NewDropPartitionMessageBuilderV1 = createNewMessageBuilderV1[*DropPartitionMessageHeader, *msgpb.DropPartitionRequest]() +) + +// createNewMessageBuilderV1 creates a new message builder with v1 marker. +func createNewMessageBuilderV1[H proto.Message, P proto.Message]() func() *mutableMesasgeBuilder[H, P] { + return func() *mutableMesasgeBuilder[H, P] { + return newMutableMessageBuilder[H, P](VersionV1) } } -// MutableMesasgeBuilder is the builder for message. -type MutableMesasgeBuilder struct { - payload []byte +// newMutableMessageBuilder creates a new builder. +// Should only used at client side. +func newMutableMessageBuilder[H proto.Message, P proto.Message](v Version) *mutableMesasgeBuilder[H, P] { + var h H + messageType := mustGetMessageTypeFromMessageHeader(h) + properties := make(propertiesImpl) + properties.Set(messageTypeKey, messageType.marshal()) + properties.Set(messageVersion, v.String()) + return &mutableMesasgeBuilder[H, P]{ + properties: properties, + } +} + +// mutableMesasgeBuilder is the builder for message. +type mutableMesasgeBuilder[H proto.Message, P proto.Message] struct { + header H + payload P properties propertiesImpl } -func (b *MutableMesasgeBuilder) WithMessageType(t MessageType) *MutableMesasgeBuilder { - b.properties.Set(messageTypeKey, t.marshal()) +// WithMessageHeader creates a new builder with determined message type. +func (b *mutableMesasgeBuilder[H, P]) WithMessageHeader(h H) *mutableMesasgeBuilder[H, P] { + b.header = h return b } // WithPayload creates a new builder with message payload. -// The MessageType is required to indicate which message type payload is. -func (b *MutableMesasgeBuilder) WithPayload(payload []byte) *MutableMesasgeBuilder { - b.payload = payload +func (b *mutableMesasgeBuilder[H, P]) WithPayload(p P) *mutableMesasgeBuilder[H, P] { + b.payload = p return b } // WithProperty creates a new builder with message property. // A key started with '_' is reserved for streaming system, should never used at user of client. -func (b *MutableMesasgeBuilder) WithProperty(key string, val string) *MutableMesasgeBuilder { +func (b *mutableMesasgeBuilder[H, P]) WithProperty(key string, val string) *mutableMesasgeBuilder[H, P] { + if b.properties.Exist(key) { + panic(fmt.Sprintf("message builder already set property field, key = %s", key)) + } b.properties.Set(key, val) return b } // WithProperties creates a new builder with message properties. // A key started with '_' is reserved for streaming system, should never used at user of client. -func (b *MutableMesasgeBuilder) WithProperties(kvs map[string]string) *MutableMesasgeBuilder { +func (b *mutableMesasgeBuilder[H, P]) WithProperties(kvs map[string]string) *mutableMesasgeBuilder[H, P] { for key, val := range kvs { b.properties.Set(key, val) } @@ -61,17 +106,28 @@ func (b *MutableMesasgeBuilder) WithProperties(kvs map[string]string) *MutableMe // BuildMutable builds a mutable message. // Panic if not set payload and message type. // should only used at client side. -func (b *MutableMesasgeBuilder) BuildMutable() MutableMessage { - if b.payload == nil { +func (b *mutableMesasgeBuilder[H, P]) BuildMutable() (MutableMessage, error) { + // payload and header must be a pointer + if reflect.ValueOf(b.header).IsNil() { + panic("message builder not ready for header field") + } + if reflect.ValueOf(b.payload).IsNil() { panic("message builder not ready for payload field") } - if !b.properties.Exist(messageTypeKey) { - panic("message builder not ready for message type field") + + // setup header. + sp, err := EncodeProto(b.header) + if err != nil { + return nil, errors.Wrap(err, "failed to encode header") + } + b.properties.Set(messageSpecialiedHeader, sp) + + payload, err := proto.Marshal(b.payload) + if err != nil { + return nil, errors.Wrap(err, "failed to marshal payload") } - // Set message version. - b.properties.Set(messageVersion, VersionV1.String()) return &messageImpl{ - payload: b.payload, + payload: payload, properties: b.properties, - } + }, nil } diff --git a/pkg/streaming/util/message/encoder.go b/pkg/streaming/util/message/encoder.go index 959619a53e..3268980bfe 100644 --- a/pkg/streaming/util/message/encoder.go +++ b/pkg/streaming/util/message/encoder.go @@ -1,6 +1,12 @@ package message -import "strconv" +import ( + "encoding/base64" + "strconv" + + "github.com/golang/protobuf/proto" + "github.com/pkg/errors" +) const base = 36 @@ -23,3 +29,21 @@ func DecodeUint64(value string) (uint64, error) { func DecodeInt64(value string) (int64, error) { return strconv.ParseInt(value, base, 64) } + +// EncodeProto encodes proto message to json string. +func EncodeProto(m proto.Message) (string, error) { + result, err := proto.Marshal(m) + if err != nil { + return "", err + } + return base64.StdEncoding.EncodeToString(result), nil +} + +// DecodeProto +func DecodeProto(data string, m proto.Message) error { + val, err := base64.StdEncoding.DecodeString(data) + if err != nil { + return errors.Wrap(err, "failed to decode base64") + } + return proto.Unmarshal(val, m) +} diff --git a/pkg/streaming/util/message/encoder_test.go b/pkg/streaming/util/message/encoder_test.go new file mode 100644 index 0000000000..18158c7301 --- /dev/null +++ b/pkg/streaming/util/message/encoder_test.go @@ -0,0 +1,27 @@ +package message + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestEncoder(t *testing.T) { + result, err := DecodeInt64(EncodeInt64(1)) + assert.NoError(t, err) + assert.Equal(t, int64(1), result) + + result2, err := DecodeUint64(EncodeUint64(1)) + assert.NoError(t, err) + assert.Equal(t, uint64(1), result2) + + result3, err := EncodeProto(&InsertMessageHeader{ + CollectionId: 1, + }) + assert.NoError(t, err) + + var result4 InsertMessageHeader + err = DecodeProto(result3, &result4) + assert.NoError(t, err) + assert.Equal(t, result4.CollectionId, int64(1)) +} diff --git a/pkg/streaming/util/message/message.go b/pkg/streaming/util/message/message.go index 2c65995067..c4c5f1e043 100644 --- a/pkg/streaming/util/message/message.go +++ b/pkg/streaming/util/message/message.go @@ -1,5 +1,7 @@ package message +import "github.com/golang/protobuf/proto" + var ( _ BasicMessage = (*messageImpl)(nil) _ MutableMessage = (*messageImpl)(nil) @@ -76,3 +78,30 @@ type ImmutableMessage interface { // MessageID returns the message id of current message. MessageID() MessageID } + +// specializedMutableMessage is the specialized mutable message interface. +type specializedMutableMessage[H proto.Message] interface { + BasicMessage + + // VChannel returns the vchannel of the message. + VChannel() string + + // TimeTick returns the time tick of the message. + TimeTick() uint64 + + // MessageHeader returns the message header. + // Modifications to the returned header will be reflected in the message. + MessageHeader() H + + // OverwriteMessageHeader overwrites the message header. + OverwriteMessageHeader(header H) +} + +// specializedImmutableMessage is the specialized immutable message interface. +type specializedImmutableMessage[H proto.Message] interface { + ImmutableMessage + + // MessageHeader returns the message header. + // Modifications to the returned header will be reflected in the message. + MessageHeader() H +} diff --git a/pkg/streaming/util/message/message_builder_test.go b/pkg/streaming/util/message/message_builder_test.go index 87ad351fbe..00f26fd6b3 100644 --- a/pkg/streaming/util/message/message_builder_test.go +++ b/pkg/streaming/util/message/message_builder_test.go @@ -1,30 +1,35 @@ package message_test import ( + "bytes" "fmt" "testing" + "github.com/golang/protobuf/proto" "github.com/stretchr/testify/assert" + "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" "github.com/milvus-io/milvus/pkg/mocks/streaming/util/mock_message" "github.com/milvus-io/milvus/pkg/streaming/util/message" ) func TestMessage(t *testing.T) { - b := message.NewMutableMessageBuilder() - mutableMessage := b. - WithMessageType(message.MessageTypeTimeTick). - WithPayload([]byte("payload")). + b := message.NewTimeTickMessageBuilderV1() + mutableMessage, err := b.WithMessageHeader(&message.TimeTickMessageHeader{}). WithProperties(map[string]string{"key": "value"}). - BuildMutable() + WithPayload(&msgpb.TimeTickMsg{}).BuildMutable() + assert.NoError(t, err) - assert.Equal(t, "payload", string(mutableMessage.Payload())) + payload, err := proto.Marshal(&message.TimeTickMessageHeader{}) + assert.NoError(t, err) + + assert.True(t, bytes.Equal(payload, mutableMessage.Payload())) assert.True(t, mutableMessage.Properties().Exist("key")) v, ok := mutableMessage.Properties().Get("key") assert.Equal(t, "value", v) assert.True(t, ok) assert.Equal(t, message.MessageTypeTimeTick, mutableMessage.MessageType()) - assert.Equal(t, 24, mutableMessage.EstimateSize()) + assert.Equal(t, 20, mutableMessage.EstimateSize()) mutableMessage.WithTimeTick(123) v, ok = mutableMessage.Properties().Get("_tt") assert.True(t, ok) @@ -96,6 +101,6 @@ func TestMessage(t *testing.T) { }) assert.Panics(t, func() { - message.NewMutableMessageBuilder().BuildMutable() + message.NewTimeTickMessageBuilderV1().BuildMutable() }) } diff --git a/pkg/streaming/util/message/message_impl.go b/pkg/streaming/util/message/message_impl.go index 7b31bcd8e7..0dc0c7da48 100644 --- a/pkg/streaming/util/message/message_impl.go +++ b/pkg/streaming/util/message/message_impl.go @@ -69,6 +69,28 @@ func (m *messageImpl) IntoImmutableMessage(id MessageID) ImmutableMessage { } } +// TimeTick returns the time tick of current message. +func (m *messageImpl) TimeTick() uint64 { + value, ok := m.properties.Get(messageTimeTick) + if !ok { + panic(fmt.Sprintf("there's a bug in the message codes, timetick lost in properties of message")) + } + tt, err := DecodeUint64(value) + if err != nil { + panic(fmt.Sprintf("there's a bug in the message codes, dirty timetick %s in properties of message", value)) + } + return tt +} + +// VChannel returns the vchannel of current message. +func (m *messageImpl) VChannel() string { + value, ok := m.properties.Get(messageVChannel) + if !ok { + panic(fmt.Sprintf("there's a bug in the message codes, vchannel lost in properties of message")) + } + return value +} + type immutableMessageImpl struct { messageImpl id MessageID @@ -92,6 +114,14 @@ func (m *immutableMessageImpl) TimeTick() uint64 { return tt } +func (m *immutableMessageImpl) VChannel() string { + value, ok := m.properties.Get(messageVChannel) + if !ok { + panic(fmt.Sprintf("there's a bug in the message codes, vchannel lost in properties of message, id: %+v", m.id)) + } + return value +} + func (m *immutableMessageImpl) LastConfirmedMessageID() MessageID { value, ok := m.properties.Get(messageLastConfirmed) if !ok { @@ -108,11 +138,3 @@ func (m *immutableMessageImpl) LastConfirmedMessageID() MessageID { func (m *immutableMessageImpl) MessageID() MessageID { return m.id } - -func (m *immutableMessageImpl) VChannel() string { - value, ok := m.properties.Get(messageVChannel) - if !ok { - panic(fmt.Sprintf("there's a bug in the message codes, vchannel lost in properties of message, id: %+v", m.id)) - } - return value -} diff --git a/pkg/streaming/util/message/messagepb/message.proto b/pkg/streaming/util/message/messagepb/message.proto new file mode 100644 index 0000000000..9783691740 --- /dev/null +++ b/pkg/streaming/util/message/messagepb/message.proto @@ -0,0 +1,84 @@ +syntax = "proto3"; + +package milvus.proto.message; + +option go_package = "github.com/milvus-io/milvus/pkg/streaming/util/message/messagepb"; + +/// +/// Message Payload Definitions +/// Some message payload is defined at msg.proto at milvus-proto for +/// compatibility. +/// 1. InsertRequest +/// 2. DeleteRequest +/// 3. TimeTickRequest +/// 4. CreateCollectionRequest +/// 5. DropCollectionRequest +/// 6. CreatePartitionRequest +/// 7. DropPartitionRequest +/// + +// FlushMessagePayload is the payload of flush message. +message FlushMessagePayload { + int64 collection_id = + 1; // indicate which the collection that segment belong to. + repeated int64 segment_id = 2; // indicate which segment to flush. +} + +/// +/// Message Header Definitions +/// Used to fast handling at streaming node write ahead. +/// The header should be simple and light enough to be parsed. +/// Do not put too much information in the header if unnecessary. +/// + +// TimeTickMessageHeader just nothing. +message TimeTickMessageHeader {} + +// InsertMessageHeader is the header of insert message. +message InsertMessageHeader { + int64 collection_id = 1; + repeated PartitionSegmentAssignment partitions = 2; +} + +// PartitionSegmentAssignment is the segment assignment of a partition. +message PartitionSegmentAssignment { + int64 partition_id = 1; + uint64 rows = 2; + uint64 binary_size = 3; + SegmentAssignment segment_assignment = 4; +} + +// SegmentAssignment is the assignment of a segment. +message SegmentAssignment { + int64 segment_id = 1; +} + +// DeleteMessageHeader +message DeleteMessageHeader { + int64 collection_id = 1; +} + +// FlushMessageHeader just nothing. +message FlushMessageHeader {} + +// CreateCollectionMessageHeader is the header of create collection message. +message CreateCollectionMessageHeader { + int64 collection_id = 1; +} + +// DropCollectionMessageHeader is the header of drop collection message. +message DropCollectionMessageHeader { + int64 collection_id = 1; +} + +// CreatePartitionMessageHeader is the header of create partition message. +message CreatePartitionMessageHeader { + int64 collection_id = 1; + int64 partition_id = 2; +} + +// DropPartitionMessageHeader is the header of drop partition message. +message DropPartitionMessageHeader { + int64 collection_id = 1; + int64 partition_id = 2; +} diff --git a/pkg/streaming/util/message/properties.go b/pkg/streaming/util/message/properties.go index 2372a22d77..551b417ea9 100644 --- a/pkg/streaming/util/message/properties.go +++ b/pkg/streaming/util/message/properties.go @@ -2,11 +2,12 @@ package message const ( // preserved properties - messageVersion = "_v" // message version for compatibility. - messageTypeKey = "_t" // message type key. - messageTimeTick = "_tt" // message time tick. - messageLastConfirmed = "_lc" // message last confirmed message id. - messageVChannel = "_vc" // message virtual channel. + messageVersion = "_v" // message version for compatibility. + messageTypeKey = "_t" // message type key. + messageTimeTick = "_tt" // message time tick. + messageLastConfirmed = "_lc" // message last confirmed message id. + messageVChannel = "_vc" // message virtual channel. + messageSpecialiedHeader = "_sh" // specialized message header. ) var ( diff --git a/pkg/streaming/util/message/specialized_message.go b/pkg/streaming/util/message/specialized_message.go new file mode 100644 index 0000000000..2864dca985 --- /dev/null +++ b/pkg/streaming/util/message/specialized_message.go @@ -0,0 +1,186 @@ +package message + +import ( + "fmt" + "reflect" + + "github.com/golang/protobuf/proto" + "github.com/pkg/errors" + + "github.com/milvus-io/milvus/pkg/streaming/util/message/messagepb" +) + +type ( + SegmentAssignment = messagepb.SegmentAssignment + PartitionSegmentAssignment = messagepb.PartitionSegmentAssignment + TimeTickMessageHeader = messagepb.TimeTickMessageHeader + InsertMessageHeader = messagepb.InsertMessageHeader + DeleteMessageHeader = messagepb.DeleteMessageHeader + CreateCollectionMessageHeader = messagepb.CreateCollectionMessageHeader + DropCollectionMessageHeader = messagepb.DropCollectionMessageHeader + CreatePartitionMessageHeader = messagepb.CreatePartitionMessageHeader + DropPartitionMessageHeader = messagepb.DropPartitionMessageHeader +) + +// messageTypeMap maps the proto message type to the message type. +var messageTypeMap = map[reflect.Type]MessageType{ + reflect.TypeOf(&TimeTickMessageHeader{}): MessageTypeTimeTick, + reflect.TypeOf(&InsertMessageHeader{}): MessageTypeInsert, + reflect.TypeOf(&DeleteMessageHeader{}): MessageTypeDelete, + reflect.TypeOf(&CreateCollectionMessageHeader{}): MessageTypeCreateCollection, + reflect.TypeOf(&DropCollectionMessageHeader{}): MessageTypeDropCollection, + reflect.TypeOf(&CreatePartitionMessageHeader{}): MessageTypeCreatePartition, + reflect.TypeOf(&DropPartitionMessageHeader{}): MessageTypeDropPartition, +} + +// List all specialized message types. +type ( + MutableTimeTickMessage = specializedMutableMessage[*TimeTickMessageHeader] + MutableInsertMessage = specializedMutableMessage[*InsertMessageHeader] + MutableDeleteMessage = specializedMutableMessage[*DeleteMessageHeader] + MutableCreateCollection = specializedMutableMessage[*CreateCollectionMessageHeader] + MutableDropCollection = specializedMutableMessage[*DropCollectionMessageHeader] + MutableCreatePartition = specializedMutableMessage[*CreatePartitionMessageHeader] + MutableDropPartition = specializedMutableMessage[*DropPartitionMessageHeader] + + ImmutableTimeTickMessage = specializedImmutableMessage[*TimeTickMessageHeader] + ImmutableInsertMessage = specializedImmutableMessage[*InsertMessageHeader] + ImmutableDeleteMessage = specializedImmutableMessage[*DeleteMessageHeader] + ImmutableCreateCollection = specializedImmutableMessage[*CreateCollectionMessageHeader] + ImmutableDropCollection = specializedImmutableMessage[*DropCollectionMessageHeader] + ImmutableCreatePartition = specializedImmutableMessage[*CreatePartitionMessageHeader] + ImmutableDropPartition = specializedImmutableMessage[*DropPartitionMessageHeader] +) + +// List all as functions for specialized messages. +var ( + AsMutableTimeTickMessage = asSpecializedMutableMessage[*TimeTickMessageHeader] + AsMutableInsertMessage = asSpecializedMutableMessage[*InsertMessageHeader] + AsMutableDeleteMessage = asSpecializedMutableMessage[*DeleteMessageHeader] + AsMutableCreateCollection = asSpecializedMutableMessage[*CreateCollectionMessageHeader] + AsMutableDropCollection = asSpecializedMutableMessage[*DropCollectionMessageHeader] + AsMutableCreatePartition = asSpecializedMutableMessage[*CreatePartitionMessageHeader] + AsMutableDropPartition = asSpecializedMutableMessage[*DropPartitionMessageHeader] + + AsImmutableTimeTickMessage = asSpecializedImmutableMessage[*TimeTickMessageHeader] + AsImmutableInsertMessage = asSpecializedImmutableMessage[*InsertMessageHeader] + AsImmutableDeleteMessage = asSpecializedImmutableMessage[*DeleteMessageHeader] + AsImmutableCreateCollection = asSpecializedImmutableMessage[*CreateCollectionMessageHeader] + AsImmutableDropCollection = asSpecializedImmutableMessage[*DropCollectionMessageHeader] + AsImmutableCreatePartition = asSpecializedImmutableMessage[*CreatePartitionMessageHeader] + AsImmutableDropPartition = asSpecializedImmutableMessage[*DropPartitionMessageHeader] +) + +// asSpecializedMutableMessage converts a MutableMessage to a specialized MutableMessage. +// Return nil, nil if the message is not the target specialized message. +// Return nil, error if the message is the target specialized message but failed to decode the specialized header. +// Return specializedMutableMessage, nil if the message is the target specialized message and successfully decoded the specialized header. +func asSpecializedMutableMessage[H proto.Message](msg MutableMessage) (specializedMutableMessage[H], error) { + underlying := msg.(*messageImpl) + + var header H + msgType := mustGetMessageTypeFromMessageHeader(header) + if underlying.MessageType() != msgType { + // The message type do not match the specialized header. + return nil, nil + } + + // Get the specialized header from the message. + val, ok := underlying.properties.Get(messageSpecialiedHeader) + if !ok { + return nil, errors.Errorf("lost specialized header, %s", msgType.String()) + } + + // Decode the specialized header. + // Must be pointer type. + t := reflect.TypeOf(header) + t.Elem() + header = reflect.New(t.Elem()).Interface().(H) + + // must be a pointer to a proto message + if err := DecodeProto(val, header); err != nil { + return nil, errors.Wrap(err, "failed to decode specialized header") + } + return &specializedMutableMessageImpl[H]{ + header: header, + messageImpl: underlying, + }, nil +} + +// asSpecializedImmutableMessage converts a ImmutableMessage to a specialized ImmutableMessage. +// Return nil, nil if the message is not the target specialized message. +// Return nil, error if the message is the target specialized message but failed to decode the specialized header. +// Return asSpecializedImmutableMessage, nil if the message is the target specialized message and successfully decoded the specialized header. +func asSpecializedImmutableMessage[H proto.Message](msg ImmutableMessage) (specializedImmutableMessage[H], error) { + underlying := msg.(*immutableMessageImpl) + + var header H + msgType := mustGetMessageTypeFromMessageHeader(header) + if underlying.MessageType() != msgType { + // The message type do not match the specialized header. + return nil, nil + } + + // Get the specialized header from the message. + val, ok := underlying.properties.Get(messageSpecialiedHeader) + if !ok { + return nil, errors.Errorf("lost specialized header, %s", msgType.String()) + } + + // Decode the specialized header. + // Must be pointer type. + t := reflect.TypeOf(header) + t.Elem() + header = reflect.New(t.Elem()).Interface().(H) + + // must be a pointer to a proto message + if err := DecodeProto(val, header); err != nil { + return nil, errors.Wrap(err, "failed to decode specialized header") + } + return &specializedImmutableMessageImpl[H]{ + header: header, + immutableMessageImpl: underlying, + }, nil +} + +// mustGetMessageTypeFromMessageHeader returns the message type of the given message header. +func mustGetMessageTypeFromMessageHeader(msg proto.Message) MessageType { + t := reflect.TypeOf(msg) + mt, ok := messageTypeMap[t] + if !ok { + panic(fmt.Sprintf("unsupported message type of proto header: %s", t.Name())) + } + return mt +} + +// specializedMutableMessageImpl is the specialized mutable message implementation. +type specializedMutableMessageImpl[H proto.Message] struct { + header H + *messageImpl +} + +// MessageHeader returns the message header. +func (m *specializedMutableMessageImpl[H]) MessageHeader() H { + return m.header +} + +// OverwriteMessageHeader overwrites the message header. +func (m *specializedMutableMessageImpl[H]) OverwriteMessageHeader(header H) { + m.header = header + newHeader, err := EncodeProto(m.header) + if err != nil { + panic(fmt.Sprintf("failed to encode insert header, there's a bug, %+v, %s", m.header, err.Error())) + } + m.messageImpl.properties.Set(messageSpecialiedHeader, newHeader) +} + +// specializedImmutableMessageImpl is the specialized immmutable message implementation. +type specializedImmutableMessageImpl[H proto.Message] struct { + header H + *immutableMessageImpl +} + +// MessageHeader returns the message header. +func (m *specializedImmutableMessageImpl[H]) MessageHeader() H { + return m.header +} diff --git a/pkg/streaming/util/message/specialized_message_test.go b/pkg/streaming/util/message/specialized_message_test.go new file mode 100644 index 0000000000..ac7f017b57 --- /dev/null +++ b/pkg/streaming/util/message/specialized_message_test.go @@ -0,0 +1,54 @@ +package message_test + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" + "github.com/milvus-io/milvus/pkg/mocks/streaming/util/mock_message" + "github.com/milvus-io/milvus/pkg/streaming/util/message" +) + +func TestAsSpecializedMessage(t *testing.T) { + m, err := message.NewInsertMessageBuilderV1(). + WithMessageHeader(&message.InsertMessageHeader{ + CollectionId: 1, + Partitions: []*message.PartitionSegmentAssignment{ + { + PartitionId: 1, + Rows: 100, + BinarySize: 1000, + }, + }, + }). + WithPayload(&msgpb.InsertRequest{}).BuildMutable() + assert.NoError(t, err) + + insertMsg, err := message.AsMutableInsertMessage(m) + assert.NoError(t, err) + assert.NotNil(t, insertMsg) + assert.Equal(t, int64(1), insertMsg.MessageHeader().CollectionId) + + h := insertMsg.MessageHeader() + h.Partitions[0].SegmentAssignment = &message.SegmentAssignment{ + SegmentId: 1, + } + insertMsg.OverwriteMessageHeader(h) + + createColMsg, err := message.AsMutableCreateCollection(m) + assert.NoError(t, err) + assert.Nil(t, createColMsg) + + m2 := m.IntoImmutableMessage(mock_message.NewMockMessageID(t)) + + insertMsg2, err := message.AsImmutableInsertMessage(m2) + assert.NoError(t, err) + assert.NotNil(t, insertMsg2) + assert.Equal(t, int64(1), insertMsg2.MessageHeader().CollectionId) + assert.Equal(t, insertMsg2.MessageHeader().Partitions[0].SegmentAssignment.SegmentId, int64(1)) + + createColMsg2, err := message.AsMutableCreateCollection(m) + assert.NoError(t, err) + assert.Nil(t, createColMsg2) +} diff --git a/pkg/streaming/util/message/test_case.go b/pkg/streaming/util/message/test_case.go new file mode 100644 index 0000000000..3ffb7707b4 --- /dev/null +++ b/pkg/streaming/util/message/test_case.go @@ -0,0 +1,148 @@ +//go:build test +// +build test + +package message + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" + "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" + "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" +) + +func CreateTestInsertMessage(t *testing.T, segmentID int64, totalRows int, timetick uint64, messageID MessageID) MutableMessage { + timestamps := make([]uint64, 0, totalRows) + for i := 0; i < totalRows; i++ { + timestamps = append(timestamps, uint64(0)) + } + rowIDs := make([]int64, 0, totalRows) + for i := 0; i < totalRows; i++ { + rowIDs = append(rowIDs, int64(i)) + } + intFieldArray := make([]int32, 0, totalRows) + for i := 0; i < totalRows; i++ { + intFieldArray = append(intFieldArray, int32(i)) + } + boolFieldArray := make([]bool, 0, totalRows) + for i := 0; i < totalRows; i++ { + boolFieldArray = append(boolFieldArray, i%2 == 0) + } + fieldsData := []*schemapb.FieldData{ + { + Type: schemapb.DataType_Int64, + FieldName: "f1", + FieldId: 1, + Field: &schemapb.FieldData_Scalars{ + Scalars: &schemapb.ScalarField{ + Data: &schemapb.ScalarField_IntData{ + IntData: &schemapb.IntArray{ + Data: intFieldArray, + }, + }, + }, + }, + }, + { + Type: schemapb.DataType_Bool, + FieldName: "f2", + FieldId: 2, + Field: &schemapb.FieldData_Scalars{ + Scalars: &schemapb.ScalarField{ + Data: &schemapb.ScalarField_BoolData{ + BoolData: &schemapb.BoolArray{ + Data: boolFieldArray, + }, + }, + }, + }, + }, + } + msg, err := NewInsertMessageBuilderV1(). + WithMessageHeader(&InsertMessageHeader{ + CollectionId: 1, + Partitions: []*PartitionSegmentAssignment{ + { + PartitionId: 2, + Rows: uint64(totalRows), + BinarySize: 10000, + SegmentAssignment: &SegmentAssignment{SegmentId: segmentID}, + }, + }, + }). + WithPayload(&msgpb.InsertRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_Insert, + Timestamp: 100, + SourceID: 1, + }, + ShardName: "v1", + DbName: "test_name", + CollectionName: "test_name", + PartitionName: "test_name", + DbID: 1, + CollectionID: 1, + PartitionID: 2, + SegmentID: 0, + Version: msgpb.InsertDataVersion_ColumnBased, + FieldsData: fieldsData, + RowIDs: rowIDs, + Timestamps: timestamps, + NumRows: uint64(totalRows), + }).BuildMutable() + if err != nil { + panic(err) + } + msg.WithVChannel("v1") + msg.WithTimeTick(timetick) + msg.WithLastConfirmed(messageID) + return msg +} + +func CreateTestCreateCollectionMessage(t *testing.T, collectionID int64, timetick uint64, messageID MessageID) MutableMessage { + header := &CreateCollectionMessageHeader{ + CollectionId: collectionID, + } + payload := &msgpb.CreateCollectionRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_CreateCollection, + MsgID: collectionID, + Timestamp: 100, + }, + DbName: "db", + CollectionName: "collection", + PartitionName: "partition", + DbID: 1, + CollectionID: collectionID, + } + + msg, err := NewCreateCollectionMessageBuilderV1(). + WithMessageHeader(header). + WithPayload(payload). + BuildMutable() + assert.NoError(t, err) + msg.WithVChannel("v1") + msg.WithTimeTick(timetick) + msg.WithLastConfirmed(messageID) + return msg +} + +// CreateTestEmptyInsertMesage creates an empty insert message for testing +func CreateTestEmptyInsertMesage(msgID int64, extraProperties map[string]string) MutableMessage { + msg, err := NewInsertMessageBuilderV1(). + WithMessageHeader(&InsertMessageHeader{}). + WithPayload(&msgpb.InsertRequest{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_Insert, + MsgID: msgID, + }, + }). + WithProperties(extraProperties). + BuildMutable() + if err != nil { + panic(err) + } + return msg +} diff --git a/pkg/streaming/walimpls/impls/pulsar/message_id.go b/pkg/streaming/walimpls/impls/pulsar/message_id.go index 59614dbe35..e2f3fa3bcd 100644 --- a/pkg/streaming/walimpls/impls/pulsar/message_id.go +++ b/pkg/streaming/walimpls/impls/pulsar/message_id.go @@ -1,7 +1,7 @@ package pulsar import ( - "encoding/hex" + "encoding/base64" "github.com/apache/pulsar-client-go/pulsar" "github.com/cockroachdb/errors" @@ -11,6 +11,12 @@ import ( var _ message.MessageID = pulsarID{} +// NewPulsarID creates a new pulsarID +// TODO: remove in future. +func NewPulsarID(id pulsar.MessageID) message.MessageID { + return pulsarID{id} +} + func UnmarshalMessageID(data string) (message.MessageID, error) { id, err := unmarshalMessageID(data) if err != nil { @@ -20,9 +26,9 @@ func UnmarshalMessageID(data string) (message.MessageID, error) { } func unmarshalMessageID(data string) (pulsarID, error) { - val, err := hex.DecodeString(data) + val, err := base64.StdEncoding.DecodeString(data) if err != nil { - return pulsarID{nil}, errors.Wrapf(message.ErrInvalidMessageID, "decode pulsar fail when decode hex with err: %s, id: %s", err.Error(), data) + return pulsarID{nil}, errors.Wrapf(message.ErrInvalidMessageID, "decode pulsar fail when decode base64 with err: %s, id: %s", err.Error(), data) } msgID, err := pulsar.DeserializeMessageID(val) if err != nil { @@ -35,6 +41,13 @@ type pulsarID struct { pulsar.MessageID } +// PulsarID returns the pulsar message id. +// Don't delete this function until conversion logic removed. +// TODO: remove in future. +func (id pulsarID) PulsarID() pulsar.MessageID { + return id.MessageID +} + func (id pulsarID) WALName() string { return walName } @@ -69,5 +82,5 @@ func (id pulsarID) EQ(other message.MessageID) bool { } func (id pulsarID) Marshal() string { - return hex.EncodeToString(id.Serialize()) + return base64.StdEncoding.EncodeToString(id.Serialize()) } diff --git a/pkg/streaming/walimpls/impls/rmq/message_id.go b/pkg/streaming/walimpls/impls/rmq/message_id.go index 51637822fd..6312cc0b3f 100644 --- a/pkg/streaming/walimpls/impls/rmq/message_id.go +++ b/pkg/streaming/walimpls/impls/rmq/message_id.go @@ -8,6 +8,12 @@ import ( var _ message.MessageID = rmqID(0) +// NewRmqID creates a new rmqID. +// TODO: remove in future. +func NewRmqID(id int64) message.MessageID { + return rmqID(id) +} + // UnmarshalMessageID unmarshal the message id. func UnmarshalMessageID(data string) (message.MessageID, error) { id, err := unmarshalMessageID(data) @@ -29,6 +35,13 @@ func unmarshalMessageID(data string) (rmqID, error) { // rmqID is the message id for rmq. type rmqID int64 +// RmqID returns the message id for conversion +// Don't delete this function until conversion logic removed. +// TODO: remove in future. +func (id rmqID) RmqID() int64 { + return int64(id) +} + // WALName returns the name of message id related wal. func (id rmqID) WALName() string { return walName diff --git a/pkg/streaming/walimpls/test_framework.go b/pkg/streaming/walimpls/test_framework.go index 7b345e94e3..3ba8c95ebd 100644 --- a/pkg/streaming/walimpls/test_framework.go +++ b/pkg/streaming/walimpls/test_framework.go @@ -14,11 +14,11 @@ import ( "testing" "time" - "github.com/golang/protobuf/proto" "github.com/remeh/sizedwaitgroup" "github.com/stretchr/testify/assert" "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" + "github.com/milvus-io/milvus-proto/go-api/v2/msgpb" "github.com/milvus-io/milvus/pkg/streaming/util/message" "github.com/milvus-io/milvus/pkg/streaming/util/options" "github.com/milvus-io/milvus/pkg/streaming/util/types" @@ -230,26 +230,11 @@ func (f *testOneWALImplsFramework) testAppend(ctx context.Context, w WALImpls) ( defer swg.Done() // ...rocksmq has a dirty implement of properties, // without commonpb.MsgHeader, it can not work. - header := commonpb.MsgHeader{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_Insert, - MsgID: int64(i), - }, - } - payload, err := proto.Marshal(&header) - if err != nil { - panic(err) - } properties := map[string]string{ "id": fmt.Sprintf("%d", i), "const": "t", } - typ := message.MessageTypeUnknown - msg := message.NewMutableMessageBuilder(). - WithMessageType(typ). - WithPayload(payload). - WithProperties(properties). - BuildMutable() + msg := message.CreateTestEmptyInsertMesage(int64(i), properties) id, err := w.Append(ctx, msg) assert.NoError(f.t, err) assert.NotNil(f.t, id) @@ -257,27 +242,20 @@ func (f *testOneWALImplsFramework) testAppend(ctx context.Context, w WALImpls) ( }(i) } swg.Wait() - // send a final hint message - header := commonpb.MsgHeader{ - Base: &commonpb.MsgBase{ - MsgType: commonpb.MsgType_Insert, - MsgID: int64(f.messageCount - 1), - }, - } - payload, err := proto.Marshal(&header) - if err != nil { - panic(err) - } + properties := map[string]string{ "id": fmt.Sprintf("%d", f.messageCount-1), "const": "t", "term": strconv.FormatInt(int64(f.term), 10), } - msg := message.NewMutableMessageBuilder(). - WithPayload(payload). - WithProperties(properties). - WithMessageType(message.MessageTypeTimeTick). - BuildMutable() + msg, err := message.NewTimeTickMessageBuilderV1().WithMessageHeader(&message.TimeTickMessageHeader{}).WithPayload(&msgpb.TimeTickMsg{ + Base: &commonpb.MsgBase{ + MsgType: commonpb.MsgType_TimeTick, + MsgID: int64(f.messageCount - 1), + }, + }).WithProperties(properties).BuildMutable() + assert.NoError(f.t, err) + id, err := w.Append(ctx, msg) assert.NoError(f.t, err) ids[f.messageCount-1] = msg.IntoImmutableMessage(id) diff --git a/pkg/util/typeutil/multipart_queue.go b/pkg/util/typeutil/multipart_queue.go new file mode 100644 index 0000000000..a51c1ebe0c --- /dev/null +++ b/pkg/util/typeutil/multipart_queue.go @@ -0,0 +1,55 @@ +package typeutil + +// NewMultipartQueue create a new multi-part queue. +func NewMultipartQueue[T any]() *MultipartQueue[T] { + return &MultipartQueue[T]{ + pendings: make([][]T, 0), + cnt: 0, + } +} + +// MultipartQueue is a multi-part queue. +type MultipartQueue[T any] struct { + pendings [][]T + cnt int +} + +// Len return the queue size. +func (pq *MultipartQueue[T]) Len() int { + return pq.cnt +} + +// AddOne add a message as pending one +func (pq *MultipartQueue[T]) AddOne(msg T) { + pq.Add([]T{msg}) +} + +// Add add a slice of message as pending one +func (pq *MultipartQueue[T]) Add(msgs []T) { + if len(msgs) == 0 { + return + } + pq.pendings = append(pq.pendings, msgs) + pq.cnt += len(msgs) +} + +// Next return the next message in pending queue. +func (pq *MultipartQueue[T]) Next() T { + if len(pq.pendings) != 0 && len(pq.pendings[0]) != 0 { + return pq.pendings[0][0] + } + var val T + return val +} + +// UnsafeAdvance do a advance without check. +// !!! Should only be called `Next` do not return nil. +func (pq *MultipartQueue[T]) UnsafeAdvance() { + if len(pq.pendings[0]) == 1 { + pq.pendings = pq.pendings[1:] + pq.cnt-- + return + } + pq.pendings[0] = pq.pendings[0][1:] + pq.cnt-- +} diff --git a/pkg/util/typeutil/multipart_queue_test.go b/pkg/util/typeutil/multipart_queue_test.go new file mode 100644 index 0000000000..3df0b26e31 --- /dev/null +++ b/pkg/util/typeutil/multipart_queue_test.go @@ -0,0 +1,20 @@ +package typeutil + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMultipartQueue(t *testing.T) { + q := NewMultipartQueue[int]() + for i := 0; i < 100; i++ { + q.AddOne(i) + assert.Equal(t, i+1, q.Len()) + } + for i := 100; i > 0; i-- { + assert.NotNil(t, q.Next()) + q.UnsafeAdvance() + assert.Equal(t, i-1, q.Len()) + } +} diff --git a/scripts/generate_proto.sh b/scripts/generate_proto.sh index 47c31e92d7..a51b1dfe2b 100755 --- a/scripts/generate_proto.sh +++ b/scripts/generate_proto.sh @@ -89,3 +89,10 @@ ${protoc_opt} --cpp_out=$CPP_SRC_DIR/src/pb cgo_msg.proto|| { echo 'generate cgo ${protoc_opt} --cpp_out=$CPP_SRC_DIR/src/pb plan.proto|| { echo 'generate plan.proto failed'; exit 1; } popd + +pushd $ROOT_DIR/pkg/streaming/util/message/messagepb + +# streaming node message protobuf +${PROTOC_BIN} --proto_path=. --go_out=plugins=grpc,paths=source_relative:. message.proto + +popd \ No newline at end of file