mirror of
https://gitee.com/milvus-io/milvus.git
synced 2025-12-08 01:58:34 +08:00
7ce876a072
This pr decoups importing segment from flush process by: 1. Exclude the importing segment from the flush policy, this approch avoids notifying the datanode to flush the importing segment, which may not exist. 2. When RootCoord call Flush, DataCoord directly set the importing segment state to `Flushed`. issue: https://github.com/milvus-io/milvus/issues/30359 --------- Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
130 lines
3.5 KiB
Go
130 lines
3.5 KiB
Go
// Licensed to the LF AI & Data foundation under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package metacache
|
|
|
|
import (
|
|
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
|
|
"github.com/milvus-io/milvus-proto/go-api/v2/msgpb"
|
|
"github.com/milvus-io/milvus/internal/proto/datapb"
|
|
"github.com/milvus-io/milvus/internal/storage"
|
|
)
|
|
|
|
const (
|
|
// NullSegment means the segment id to discard
|
|
// happens when segment compacted to 0 lines and target segment is dropped directly
|
|
NullSegment = int64(-1)
|
|
)
|
|
|
|
type SegmentInfo struct {
|
|
segmentID int64
|
|
partitionID int64
|
|
state commonpb.SegmentState
|
|
startPosition *msgpb.MsgPosition
|
|
checkpoint *msgpb.MsgPosition
|
|
startPosRecorded bool
|
|
flushedRows int64
|
|
bufferRows int64
|
|
syncingRows int64
|
|
bfs *BloomFilterSet
|
|
compactTo int64
|
|
level datapb.SegmentLevel
|
|
syncingTasks int32
|
|
}
|
|
|
|
func (s *SegmentInfo) SegmentID() int64 {
|
|
return s.segmentID
|
|
}
|
|
|
|
func (s *SegmentInfo) PartitionID() int64 {
|
|
return s.partitionID
|
|
}
|
|
|
|
func (s *SegmentInfo) State() commonpb.SegmentState {
|
|
return s.state
|
|
}
|
|
|
|
// NumOfRows returns sum of number of rows,
|
|
// including flushed, syncing and buffered
|
|
func (s *SegmentInfo) NumOfRows() int64 {
|
|
return s.flushedRows + s.syncingRows + s.bufferRows
|
|
}
|
|
|
|
// FlushedRows return flushed rows number.
|
|
func (s *SegmentInfo) FlushedRows() int64 {
|
|
return s.flushedRows
|
|
}
|
|
|
|
func (s *SegmentInfo) StartPosition() *msgpb.MsgPosition {
|
|
return s.startPosition
|
|
}
|
|
|
|
func (s *SegmentInfo) Checkpoint() *msgpb.MsgPosition {
|
|
return s.checkpoint
|
|
}
|
|
|
|
func (s *SegmentInfo) GetHistory() []*storage.PkStatistics {
|
|
return s.bfs.GetHistory()
|
|
}
|
|
|
|
func (s *SegmentInfo) CompactTo() int64 {
|
|
return s.compactTo
|
|
}
|
|
|
|
func (s *SegmentInfo) GetBloomFilterSet() *BloomFilterSet {
|
|
return s.bfs
|
|
}
|
|
|
|
func (s *SegmentInfo) Level() datapb.SegmentLevel {
|
|
return s.level
|
|
}
|
|
|
|
func (s *SegmentInfo) Clone() *SegmentInfo {
|
|
return &SegmentInfo{
|
|
segmentID: s.segmentID,
|
|
partitionID: s.partitionID,
|
|
state: s.state,
|
|
startPosition: s.startPosition,
|
|
checkpoint: s.checkpoint,
|
|
startPosRecorded: s.startPosRecorded,
|
|
flushedRows: s.flushedRows,
|
|
bufferRows: s.bufferRows,
|
|
syncingRows: s.syncingRows,
|
|
bfs: s.bfs,
|
|
compactTo: s.compactTo,
|
|
level: s.level,
|
|
syncingTasks: s.syncingTasks,
|
|
}
|
|
}
|
|
|
|
func NewSegmentInfo(info *datapb.SegmentInfo, bfs *BloomFilterSet) *SegmentInfo {
|
|
level := info.GetLevel()
|
|
if level == datapb.SegmentLevel_Legacy {
|
|
level = datapb.SegmentLevel_L1
|
|
}
|
|
return &SegmentInfo{
|
|
segmentID: info.GetID(),
|
|
partitionID: info.GetPartitionID(),
|
|
state: info.GetState(),
|
|
flushedRows: info.GetNumOfRows(),
|
|
startPosition: info.GetStartPosition(),
|
|
checkpoint: info.GetDmlPosition(),
|
|
startPosRecorded: true,
|
|
level: level,
|
|
bfs: bfs,
|
|
}
|
|
}
|