admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-07 01:28:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update repack delete msg (#8631)

Browse Source 
Signed-off-by: yudong.cai <yudong.cai@zilliz.com>
This commit is contained in:
Cai Yudong 2021-09-27 10:35:58 +08:00 committed by GitHub
parent b23761f3ed
commit ece0a06798
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 98 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/msgstream/mq_msgstream.go
View File

@ -217,8 +217,8 @@ func (ms *mqMsgStream) Produce(msgPack *MsgPack) error {
switch msgType { switch msgType {
case commonpb.MsgType_Insert: case commonpb.MsgType_Insert:
result, err = InsertRepackFunc(tsMsgs, reBucketValues) result, err = InsertRepackFunc(tsMsgs, reBucketValues)
//case commonpb.MsgType_Delete: case commonpb.MsgType_Delete:
// result, err = DeleteRepackFunc(tsMsgs, reBucketValues) result, err = DeleteRepackFunc(tsMsgs, reBucketValues)
default: default:
result, err = DefaultRepackFunc(tsMsgs, reBucketValues) result, err = DefaultRepackFunc(tsMsgs, reBucketValues)
} }

106
internal/msgstream/mq_msgstream_test.go
View File

@ -302,59 +302,59 @@ func TestStream_PulsarMsgStream_InsertRepackFunc(t *testing.T) {
(*outputStream).Close() (*outputStream).Close()
} }
//func TestStream_PulsarMsgStream_DeleteRepackFunc(t *testing.T) { func TestStream_PulsarMsgStream_DeleteRepackFunc(t *testing.T) {
// pulsarAddress, _ := Params.Load("_PulsarAddress") pulsarAddress, _ := Params.Load("_PulsarAddress")
// c1, c2 := funcutil.RandomString(8), funcutil.RandomString(8) c1, c2 := funcutil.RandomString(8), funcutil.RandomString(8)
// producerChannels := []string{c1, c2} producerChannels := []string{c1, c2}
// consumerChannels := []string{c1, c2} consumerChannels := []string{c1, c2}
// consumerSubName := funcutil.RandomString(8) consumerSubName := funcutil.RandomString(8)
//
// baseMsg := BaseMsg{ baseMsg := BaseMsg{
// BeginTimestamp: 0, BeginTimestamp: 0,
// EndTimestamp: 0, EndTimestamp: 0,
// HashValues: []uint32{1, 3}, HashValues: []uint32{1},
// } }
//
// deleteRequest := internalpb.DeleteRequest{ deleteRequest := internalpb.DeleteRequest{
// Base: &commonpb.MsgBase{ Base: &commonpb.MsgBase{
// MsgType: commonpb.MsgType_Delete, MsgType: commonpb.MsgType_Delete,
// MsgID: 1, MsgID: 1,
// Timestamp: 1, Timestamp: 1,
// SourceID: 1, SourceID: 1,
// }, },
// CollectionName: "Collection", CollectionName: "Collection",
// ChannelID: "1", ChannelID: "1",
// Timestamps: []Timestamp{1, 1}, Timestamp: Timestamp(1),
// PrimaryKeys: []int64{1, 3}, ExprPlan: []byte{},
// } }
// deleteMsg := &DeleteMsg{ deleteMsg := &DeleteMsg{
// BaseMsg: baseMsg, BaseMsg: baseMsg,
// DeleteRequest: deleteRequest, DeleteRequest: deleteRequest,
// } }
//
// msgPack := MsgPack{} msgPack := MsgPack{}
// msgPack.Msgs = append(msgPack.Msgs, deleteMsg) msgPack.Msgs = append(msgPack.Msgs, deleteMsg)
//
// factory := ProtoUDFactory{} factory := ProtoUDFactory{}
// pulsarClient, _ := mqclient.GetPulsarClientInstance(pulsar.ClientOptions{URL: pulsarAddress}) pulsarClient, _ := mqclient.GetPulsarClientInstance(pulsar.ClientOptions{URL: pulsarAddress})
// inputStream, _ := NewMqMsgStream(context.Background(), 100, 100, pulsarClient, factory.NewUnmarshalDispatcher()) inputStream, _ := NewMqMsgStream(context.Background(), 100, 100, pulsarClient, factory.NewUnmarshalDispatcher())
// inputStream.AsProducer(producerChannels) inputStream.AsProducer(producerChannels)
// inputStream.Start() inputStream.Start()
//
// pulsarClient2, _ := mqclient.GetPulsarClientInstance(pulsar.ClientOptions{URL: pulsarAddress}) pulsarClient2, _ := mqclient.GetPulsarClientInstance(pulsar.ClientOptions{URL: pulsarAddress})
// outputStream, _ := NewMqMsgStream(context.Background(), 100, 100, pulsarClient2, factory.NewUnmarshalDispatcher()) outputStream, _ := NewMqMsgStream(context.Background(), 100, 100, pulsarClient2, factory.NewUnmarshalDispatcher())
// outputStream.AsConsumer(consumerChannels, consumerSubName) outputStream.AsConsumer(consumerChannels, consumerSubName)
// outputStream.Start() outputStream.Start()
// var output MsgStream = outputStream var output MsgStream = outputStream
//
// err := (*inputStream).Produce(&msgPack) err := (*inputStream).Produce(&msgPack)
// if err != nil { if err != nil {
// log.Fatalf("produce error = %v", err) log.Fatalf("produce error = %v", err)
// } }
// receiveMsg(output, len(msgPack.Msgs)*2) receiveMsg(output, len(msgPack.Msgs)*1)
// (*inputStream).Close() (*inputStream).Close()
// (*outputStream).Close() (*outputStream).Close()
//} }
func TestStream_PulsarMsgStream_DefaultRepackFunc(t *testing.T) { func TestStream_PulsarMsgStream_DefaultRepackFunc(t *testing.T) {
pulsarAddress, _ := Params.Load("_PulsarAddress") pulsarAddress, _ := Params.Load("_PulsarAddress")

91
internal/msgstream/repack_func.go
View File

@ -74,54 +74,49 @@ func InsertRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, e
return result, nil return result, nil
} }
//func DeleteRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, error) { func DeleteRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, error) {
// result := make(map[int32]*MsgPack) result := make(map[int32]*MsgPack)
// for i, request := range tsMsgs { for i, request := range tsMsgs {
// if request.Type() != commonpb.MsgType_Delete { if request.Type() != commonpb.MsgType_Delete {
// return nil, errors.New("msg's must be Delete") return nil, errors.New("msg's must be Delete")
// } }
// deleteRequest := request.(*DeleteMsg) deleteRequest := request.(*DeleteMsg)
// keys := hashKeys[i] keys := hashKeys[i]
//
// timestampLen := len(deleteRequest.Timestamps) if len(keys) != 1 {
// primaryKeysLen := len(deleteRequest.PrimaryKeys) return nil, errors.New("len(msg.hashValue) must equal 1, but it is: " + strconv.Itoa(len(keys)))
// keysLen := len(keys) }
//
// if keysLen != timestampLen || keysLen != primaryKeysLen { key := keys[0]
// return nil, errors.New("the length of hashValue, timestamps, primaryKeys are not equal") _, ok := result[key]
// } if !ok {
// msgPack := MsgPack{}
// for index, key := range keys { result[key] = &msgPack
// _, ok := result[key] }
// if !ok {
// msgPack := MsgPack{} sliceRequest := internalpb.DeleteRequest{
// result[key] = &msgPack Base: &commonpb.MsgBase{
// } MsgType: commonpb.MsgType_Delete,
// MsgID: deleteRequest.Base.MsgID,
// sliceRequest := internalpb.DeleteRequest{ Timestamp: deleteRequest.Timestamp,
// Base: &commonpb.MsgBase{ SourceID: deleteRequest.Base.SourceID,
// MsgType: commonpb.MsgType_Delete, },
// MsgID: deleteRequest.Base.MsgID, CollectionName: deleteRequest.CollectionName,
// Timestamp: deleteRequest.Timestamps[index], ChannelID: deleteRequest.ChannelID,
// SourceID: deleteRequest.Base.SourceID, Timestamp: deleteRequest.Timestamp,
// }, ExprPlan: deleteRequest.ExprPlan,
// CollectionName: deleteRequest.CollectionName, }
// ChannelID: deleteRequest.ChannelID,
// Timestamps: []uint64{deleteRequest.Timestamps[index]}, deleteMsg := &DeleteMsg{
// PrimaryKeys: []int64{deleteRequest.PrimaryKeys[index]}, BaseMsg: BaseMsg{
// } Ctx: request.TraceCtx(),
// },
// deleteMsg := &DeleteMsg{ DeleteRequest: sliceRequest,
// BaseMsg: BaseMsg{ }
// Ctx: request.TraceCtx(), result[key].Msgs = append(result[key].Msgs, deleteMsg)
// }, }
// DeleteRequest: sliceRequest, return result, nil
// } }
// result[key].Msgs = append(result[key].Msgs, deleteMsg)
// }
// }
// return result, nil
//}
func DefaultRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, error) { func DefaultRepackFunc(tsMsgs []TsMsg, hashKeys [][]int32) (map[int32]*MsgPack, error) {
result := make(map[int32]*MsgPack) result := make(map[int32]*MsgPack)