mirror of
https://gitee.com/milvus-io/milvus.git
synced 2025-12-07 01:28:27 +08:00
84 lines
2.5 KiB
Go
84 lines
2.5 KiB
Go
package controller
|
|
|
|
import (
|
|
"fmt"
|
|
"strconv"
|
|
"time"
|
|
|
|
"github.com/zilliztech/milvus-distributed/internal/conf"
|
|
"github.com/zilliztech/milvus-distributed/internal/proto/internalpb"
|
|
"github.com/zilliztech/milvus-distributed/internal/master/collection"
|
|
"github.com/zilliztech/milvus-distributed/internal/master/id"
|
|
"github.com/zilliztech/milvus-distributed/internal/master/informer"
|
|
"github.com/zilliztech/milvus-distributed/internal/master/kv"
|
|
"github.com/zilliztech/milvus-distributed/internal/master/segment"
|
|
)
|
|
|
|
func SegmentStatsController(kvbase kv.Base, errch chan error) {
|
|
ssChan := make(chan internalpb.SegmentStatistics, 10)
|
|
ssClient := informer.NewPulsarClient()
|
|
go segment.Listener(ssChan, ssClient)
|
|
for {
|
|
select {
|
|
case ss := <-ssChan:
|
|
errch <- ComputeCloseTime(ss, kvbase)
|
|
case <-time.After(5 * time.Second):
|
|
fmt.Println("wait for new request")
|
|
return
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
func ComputeCloseTime(ss internalpb.SegmentStatistics, kvbase kv.Base) error {
|
|
if int(ss.MemorySize) > int(conf.Config.Master.SegmentThreshole*0.8) {
|
|
currentTime := time.Now()
|
|
//memRate := int(ss.MemoryRate)
|
|
memRate := 1 // to do
|
|
if memRate == 0 {
|
|
memRate = 1
|
|
}
|
|
sec := int(conf.Config.Master.SegmentThreshole*0.2) / memRate
|
|
data, err := kvbase.Load("segment/" + strconv.Itoa(int(ss.SegmentId)))
|
|
if err != nil {
|
|
return err
|
|
}
|
|
seg, err := segment.JSON2Segment(data)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
seg.CloseTimeStamp = uint64(currentTime.Add(time.Duration(sec) * time.Second).Unix())
|
|
fmt.Println(seg)
|
|
updateData, err := segment.Segment2JSON(*seg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
kvbase.Save("segment/"+strconv.Itoa(int(ss.SegmentId)), updateData)
|
|
//create new segment
|
|
newSegID := id.New().Uint64()
|
|
newSeg := segment.NewSegment(newSegID, seg.CollectionID, seg.CollectionName, "default", seg.ChannelStart, seg.ChannelEnd, currentTime, time.Unix(1<<36-1, 0))
|
|
newSegData, err := segment.Segment2JSON(*&newSeg)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
//save to kv store
|
|
kvbase.Save("segment/"+strconv.Itoa(int(newSegID)), newSegData)
|
|
// update collection data
|
|
c, _ := kvbase.Load("collection/" + strconv.Itoa(int(seg.CollectionID)))
|
|
collectionMeta, err := collection.JSON2Collection(c)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
segIDs := collectionMeta.SegmentIDs
|
|
segIDs = append(segIDs, newSegID)
|
|
collectionMeta.SegmentIDs = segIDs
|
|
cData, err := collection.Collection2JSON(*collectionMeta)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
kvbase.Save("segment/"+strconv.Itoa(int(seg.CollectionID)), cData)
|
|
}
|
|
return nil
|
|
}
|
|
|