congqixia 0b905078e7
Use writebuffer, sync manager refactory in datanode (#28320)
See also #27675
This PR make previously merged refactory of datanode go online
- Use write node to replace insert/delete node
- Use write buffer manager to control all buffers
- Use sync manager to control sync tasks instead of flush manager

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
2023-11-15 15:24:18 +08:00

71 lines
1.8 KiB
Go

package writebuffer
import (
"time"
"github.com/milvus-io/milvus/internal/allocator"
"github.com/milvus-io/milvus/internal/datanode/metacache"
"github.com/milvus-io/milvus/internal/datanode/syncmgr"
"github.com/milvus-io/milvus/pkg/util/paramtable"
)
const (
// DeletePolicyBFPKOracle is the const config value for using bf pk oracle as delete policy
DeletePolicyBFPkOracle = `bloom_filter_pkoracle`
// DeletePolicyL0Delta is the const config value for using L0 delta as deleta policy.
DeletePolicyL0Delta = `l0_delta`
)
type WriteBufferOption func(opt *writeBufferOption)
type writeBufferOption struct {
deletePolicy string
idAllocator allocator.Interface
syncPolicies []SyncPolicy
pkStatsFactory metacache.PkStatsFactory
metaWriter syncmgr.MetaWriter
}
func defaultWBOption() *writeBufferOption {
return &writeBufferOption{
// TODO use l0 delta as default after implementation.
deletePolicy: paramtable.Get().DataNodeCfg.DeltaPolicy.GetValue(),
syncPolicies: []SyncPolicy{
SyncFullBuffer,
GetSyncStaleBufferPolicy(paramtable.Get().DataNodeCfg.SyncPeriod.GetAsDuration(time.Second)),
},
}
}
func WithDeletePolicy(policy string) WriteBufferOption {
return func(opt *writeBufferOption) {
opt.deletePolicy = policy
}
}
func WithIDAllocator(allocator allocator.Interface) WriteBufferOption {
return func(opt *writeBufferOption) {
opt.idAllocator = allocator
}
}
func WithPKStatsFactory(factory metacache.PkStatsFactory) WriteBufferOption {
return func(opt *writeBufferOption) {
opt.pkStatsFactory = factory
}
}
func WithMetaWriter(writer syncmgr.MetaWriter) WriteBufferOption {
return func(opt *writeBufferOption) {
opt.metaWriter = writer
}
}
func WithSyncPolicy(policy SyncPolicy) WriteBufferOption {
return func(opt *writeBufferOption) {
opt.syncPolicies = append(opt.syncPolicies, policy)
}
}