mirror of
https://gitee.com/milvus-io/milvus.git
synced 2025-12-07 09:38:39 +08:00
0b905078e7
See also #27675 This PR make previously merged refactory of datanode go online - Use write node to replace insert/delete node - Use write buffer manager to control all buffers - Use sync manager to control sync tasks instead of flush manager Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
63 lines
2.0 KiB
Go
63 lines
2.0 KiB
Go
package writebuffer
|
|
|
|
import (
|
|
"time"
|
|
|
|
"github.com/samber/lo"
|
|
"go.uber.org/atomic"
|
|
|
|
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
|
|
"github.com/milvus-io/milvus/internal/datanode/metacache"
|
|
"github.com/milvus-io/milvus/pkg/util/tsoutil"
|
|
"github.com/milvus-io/milvus/pkg/util/typeutil"
|
|
)
|
|
|
|
type SyncPolicy func(buffers []*segmentBuffer, ts typeutil.Timestamp) []int64
|
|
|
|
func SyncFullBuffer(buffers []*segmentBuffer, _ typeutil.Timestamp) []int64 {
|
|
return lo.FilterMap(buffers, func(buf *segmentBuffer, _ int) (int64, bool) {
|
|
return buf.segmentID, buf.IsFull()
|
|
})
|
|
}
|
|
|
|
func GetSyncStaleBufferPolicy(staleDuration time.Duration) SyncPolicy {
|
|
return func(buffers []*segmentBuffer, ts typeutil.Timestamp) []int64 {
|
|
current := tsoutil.PhysicalTime(ts)
|
|
return lo.FilterMap(buffers, func(buf *segmentBuffer, _ int) (int64, bool) {
|
|
minTs := buf.MinTimestamp()
|
|
start := tsoutil.PhysicalTime(minTs)
|
|
|
|
return buf.segmentID, current.Sub(start) > staleDuration
|
|
})
|
|
}
|
|
}
|
|
|
|
func GetFlushingSegmentsPolicy(meta metacache.MetaCache) SyncPolicy {
|
|
return func(_ []*segmentBuffer, _ typeutil.Timestamp) []int64 {
|
|
return meta.GetSegmentIDsBy(metacache.WithSegmentState(commonpb.SegmentState_Flushing))
|
|
}
|
|
}
|
|
|
|
func GetFlushTsPolicy(flushTimestamp *atomic.Uint64, meta metacache.MetaCache) SyncPolicy {
|
|
return func(buffers []*segmentBuffer, ts typeutil.Timestamp) []int64 {
|
|
flushTs := flushTimestamp.Load()
|
|
if flushTs != nonFlushTS && ts >= flushTs {
|
|
// flush segment start pos < flushTs && checkpoint > flushTs
|
|
ids := lo.FilterMap(buffers, func(buf *segmentBuffer, _ int) (int64, bool) {
|
|
seg, ok := meta.GetSegmentByID(buf.segmentID)
|
|
if !ok {
|
|
return buf.segmentID, false
|
|
}
|
|
return buf.segmentID, seg.State() == commonpb.SegmentState_Flushed && buf.MinTimestamp() < flushTs
|
|
})
|
|
// set segment flushing
|
|
meta.UpdateSegments(metacache.UpdateState(commonpb.SegmentState_Flushing),
|
|
metacache.WithSegmentIDs(ids...), metacache.WithSegmentState(commonpb.SegmentState_Growing))
|
|
|
|
// flush all buffer
|
|
return ids
|
|
}
|
|
return nil
|
|
}
|
|
}
|