milvus/writer/message_client/message_client.go

package message_client

import (
	"context"
	"github.com/apache/pulsar/pulsar-client-go/pulsar"
	"github.com/golang/protobuf/proto"
	"log"
	"writer/pb"
)


type MessageClient struct {

	//message channel
	insertOrDeleteChan chan *pb.InsertOrDeleteMsg
	searchChan chan *pb.SearchMsg
	timeSyncChan chan *pb.TimeSyncMsg

	// pulsar
	client pulsar.Client
	key2segProducer pulsar.Producer
	writeSyncProducer pulsar.Producer
	insertOrDeleteConsumer pulsar.Consumer
	searchConsumer pulsar.Consumer
	timeSyncConsumer pulsar.Consumer

	// batch messages
	InsertMsg  []*pb.InsertOrDeleteMsg
	DeleteMsg  []*pb.InsertOrDeleteMsg
	SearchMsg  []*pb.SearchMsg
	timeSyncMsg    []*pb.TimeSyncMsg
}

func (mc *MessageClient)ReceiveInsertOrDeleteMsg() {
	for {
		insetOrDeleteMsg := pb.InsertOrDeleteMsg{}
		msg, err := mc.insertOrDeleteConsumer.Receive(context.Background())
		err = proto.Unmarshal(msg.Payload(), &insetOrDeleteMsg)
		if err != nil {
			log.Fatal(err)
		}
		mc.insertOrDeleteChan <- &insetOrDeleteMsg
	}
}

func (mc *MessageClient)ReceiveSearchMsg() {
	for {
		searchMsg := pb.SearchMsg{}
		msg, err := mc.insertOrDeleteConsumer.Receive(context.Background())
		err = proto.Unmarshal(msg.Payload(), &searchMsg)
		if err != nil {
			log.Fatal(err)
		}
		mc.searchChan <- &searchMsg
	}
}

func (mc *MessageClient)ReceiveTimeSyncMsg() {
	for {
		timeSyncMsg := pb.TimeSyncMsg{}
		msg, err := mc.insertOrDeleteConsumer.Receive(context.Background())
		err = proto.Unmarshal(msg.Payload(), &timeSyncMsg)
		if err != nil {
			log.Fatal(err)
		}
		mc.timeSyncChan <- &timeSyncMsg
	}
}

func (mc *MessageClient) ReceiveMessage() {
	go mc.ReceiveInsertOrDeleteMsg()
	go mc.ReceiveSearchMsg()
	go mc.ReceiveTimeSyncMsg()
}

func (mc *MessageClient) CreatProducer(topicName string) pulsar.Producer{
	producer, err := mc.client.CreateProducer(pulsar.ProducerOptions{
		Topic: topicName,
	})

	if err != nil {
		log.Fatal(err)
	}
	return producer
}

func (mc *MessageClient) CreateConsumer(topicName string) pulsar.Consumer {
	consumer, err := mc.client.Subscribe(pulsar.ConsumerOptions{
		Topic:           topicName,
		SubscriptionName: "multi-topic-sub",
	})

	if err != nil {
		log.Fatal(err)
	}
	return consumer
}

func (mc *MessageClient) CreateClient(url string) pulsar.Client {
	// create client
	client, err := pulsar.NewClient(pulsar.ClientOptions{
		URL: url,
	})

	if err != nil {
		log.Fatal(err)
	}
	return client
}

func (mc *MessageClient) InitClient(url string) {
	//create client
	mc.client = mc.CreateClient(url)

	//create producer
	mc.key2segProducer = mc.CreatProducer("Key2Seg")
	mc.writeSyncProducer = mc.CreatProducer("TimeSync")

	//create consumer
	mc.insertOrDeleteConsumer = mc.CreateConsumer("InsertOrDelete")
	mc.searchConsumer = mc.CreateConsumer("Search")
	mc.timeSyncConsumer = mc.CreateConsumer("TimeSync")

	// init channel
	mc.insertOrDeleteChan = make(chan *pb.InsertOrDeleteMsg, 1000)
	mc.searchChan  = make(chan *pb.SearchMsg, 1000)
	mc.timeSyncChan = make(chan *pb.TimeSyncMsg, 1000)
}

func (mc *MessageClient) Close() {
	defer mc.client.Close()
	defer mc.key2segProducer.Close()
	defer mc.writeSyncProducer.Close()
	defer mc.insertOrDeleteConsumer.Close()
	defer mc.searchConsumer.Close()
	defer mc.timeSyncConsumer.Close()
}

type JobType int
const (
	OpInQueryNode JobType = 0
	OpInWriteNode JobType = 1
)

type MessageType int
const (
	InsertOrDelete MessageType = 0
	Delete         MessageType = 1
	Search         MessageType = 2
	TimeSync       MessageType = 3
	Key2Seg        MessageType = 4
	Statistics     MessageType = 5
)

func (mc *MessageClient) PrepareMsg(messageType MessageType, msgLen int) {
	switch messageType {
	case InsertOrDelete:
		for i := 0; i < msgLen; i++ {
			msg := <-mc.insertOrDeleteChan
			if msg.Op == pb.OpType_INSERT {
				mc.InsertMsg[i] = msg
			} else {
				mc.DeleteMsg[i] = msg
			}
		}
	case Search:
		for i := 0; i < msgLen; i++ {
			msg := <-mc.searchChan
			mc.SearchMsg[i] = msg
		}
	case TimeSync:
		for i := 0; i < msgLen; i++ {
			msg := <-mc.timeSyncChan
			mc.timeSyncMsg[i] = msg
		}
	}
}

func (mc *MessageClient)PrepareBatchMsg() []int{
	// assume the channel not full
	mc.InsertMsg = make([]*pb.InsertOrDeleteMsg, 1000)
	mc.DeleteMsg = make([]*pb.InsertOrDeleteMsg, 1000)
	mc.SearchMsg = make([]*pb.SearchMsg, 1000)
	mc.timeSyncMsg = make([]*pb.TimeSyncMsg, 1000)

	// get the length of every channel
	insertOrDeleteLen := len(mc.insertOrDeleteChan)
	searchLen := len(mc.searchChan)
	timeLen := len(mc.timeSyncChan)

	// get message from channel to slice
	mc.PrepareMsg(InsertOrDelete, insertOrDeleteLen)
	mc.PrepareMsg(Search, searchLen)
	mc.PrepareMsg(TimeSync, timeLen)

	return []int{insertOrDeleteLen, searchLen, timeLen}
}