package proxy import ( "context" "fmt" "github.com/apache/pulsar-client-go/pulsar" "github.com/zilliztech/milvus-distributed/internal/allocator" "github.com/zilliztech/milvus-distributed/internal/conf" "github.com/zilliztech/milvus-distributed/internal/proto/commonpb" "github.com/zilliztech/milvus-distributed/internal/proto/internalpb" "github.com/zilliztech/milvus-distributed/internal/util/typeutil" etcd "go.etcd.io/etcd/clientv3" "strconv" ) type BaseRequest interface { Type() internalpb.ReqType PreExecute() commonpb.Status Execute() commonpb.Status PostExecute() commonpb.Status WaitToFinish() commonpb.Status } type ProxyOptions struct { //proxy server address string //grpc server address master_address string //master server addess collectionMetaRootPath string // etcd root path,read metas of collections and segments from etcd pulsarAddr string // pulsar address for reader readerTopicsPrefix string numReadTopics int deleteTopic string queryTopic string resultTopic string resultGroup string numReaderNode int proxyId int64 //start from 1 etcdEndpoints []string //timestamporacle tsoRootPath string //etcd root path, store timestamp into this key tsoSaveInterval uint64 //timetick timeTickInterval uint64 timeTickTopic string timeTickPeerId int64 //start from 1 // inner member proxyServer *proxyServer tso *allocator.TimestampAllocator timeTick *timeTick ctx context.Context cancel context.CancelFunc } func ReadProxyOptionsFromConfig() (*ProxyOptions, error) { conf.LoadConfig("config.yaml") etcdRootPath := conf.Config.Etcd.Rootpath if etcdRootPath[len(etcdRootPath)-1] == '/' { etcdRootPath = etcdRootPath[0 : len(etcdRootPath)-1] } return &ProxyOptions{ address: conf.Config.Proxy.Network.Address + ":" + strconv.Itoa(conf.Config.Proxy.Network.Port), master_address: conf.Config.Master.Address + ":" + strconv.Itoa(int(conf.Config.Master.Port)), collectionMetaRootPath: etcdRootPath, pulsarAddr: "pulsar://" + conf.Config.Pulsar.Address + ":" + strconv.Itoa(int(conf.Config.Pulsar.Port)), readerTopicsPrefix: conf.Config.Proxy.PulsarTopics.ReaderTopicPrefix, numReadTopics: conf.Config.Proxy.PulsarTopics.NumReaderTopics, deleteTopic: conf.Config.Proxy.PulsarTopics.DeleteTopic, queryTopic: conf.Config.Proxy.PulsarTopics.QueryTopic, resultTopic: conf.Config.Proxy.PulsarTopics.ResultTopic, resultGroup: conf.Config.Proxy.PulsarTopics.ResultGroup, numReaderNode: conf.Config.Proxy.NumReaderNodes, proxyId: int64(conf.Config.Proxy.ProxyId), etcdEndpoints: []string{conf.Config.Etcd.Address + ":" + strconv.Itoa(int(conf.Config.Etcd.Port))}, tsoRootPath: etcdRootPath, tsoSaveInterval: uint64(conf.Config.Proxy.TosSaveInterval), timeTickInterval: uint64(conf.Config.Proxy.TimeTickInterval), timeTickTopic: conf.Config.Proxy.PulsarTopics.TimeTickTopic, timeTickPeerId: int64(conf.Config.Proxy.ProxyId), }, nil } func StartProxy(opt *ProxyOptions) error { //global context opt.ctx, opt.cancel = context.WithCancel(context.Background()) ///////////////////// timestamporacle ////////////////////////// //etcdTso, err := etcd.New(etcd.Config{Endpoints: opt.etcdEndpoints}) //if err != nil { // return err //} //tso := ×tampOracle{ // client: etcdTso, // ctx: opt.ctx, // rootPath: opt.tsoRootPath, // saveInterval: opt.tsoSaveInterval, //} //tso.Restart(opt.proxyId) tso := allocator.NewTimestampAllocator() /////////////////// proxy server /////////////////////////////// //readerTopics, send insert and delete message into these topics readerTopics := make([]string, 0, opt.numReadTopics) for i := 0; i < opt.numReadTopics; i++ { readerTopics = append(readerTopics, opt.readerTopicsPrefix+strconv.Itoa(i)) } etcdProxy, err := etcd.New(etcd.Config{Endpoints: opt.etcdEndpoints}) if err != nil { return err } srv := &proxyServer{ address: opt.address, masterAddress: opt.master_address, rootPath: opt.collectionMetaRootPath, pulsarAddr: opt.pulsarAddr, readerTopics: readerTopics, deleteTopic: opt.deleteTopic, queryTopic: opt.queryTopic, resultTopic: opt.resultTopic, resultGroup: opt.resultTopic, numReaderNode: opt.numReaderNode, proxyId: opt.proxyId, getTimestamp: tso.Alloc, client: etcdProxy, ctx: opt.ctx, } err = startProxyServer(srv) if err != nil { return err } ////////////////////////// time tick ///////////////////////////////// ttClient, err := pulsar.NewClient(pulsar.ClientOptions{URL: opt.pulsarAddr}) if err != nil { return err } ttProducer, err := ttClient.CreateProducer(pulsar.ProducerOptions{Topic: opt.timeTickTopic}) if err != nil { return err } tt := &timeTick{ interval: opt.timeTickInterval, pulsarProducer: ttProducer, peer_id: opt.timeTickPeerId, ctx: opt.ctx, areRequestsDelivered: func(ts typeutil.Timestamp) bool { return srv.reqSch.AreRequestsDelivered(ts, 2) }, getTimestamp: func() (typeutil.Timestamp, error) { ts, st := tso.AllocOne() return ts, st }, } err = tt.Restart() if err != nil { return fmt.Errorf("timeTick Restart Failed") } opt.proxyServer = srv opt.tso = tso opt.timeTick = tt srv.wg.Wait() return nil }