milvus/internal/querycoordv2/checkers/controller.go

// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package checkers

import (
	"context"
	"sync"
	"time"

	"go.uber.org/zap"

	"github.com/milvus-io/milvus/internal/querycoordv2/balance"
	"github.com/milvus-io/milvus/internal/querycoordv2/meta"
	. "github.com/milvus-io/milvus/internal/querycoordv2/params"
	"github.com/milvus-io/milvus/internal/querycoordv2/session"
	"github.com/milvus-io/milvus/internal/querycoordv2/task"
	"github.com/milvus-io/milvus/pkg/log"
)

var checkRoundTaskNumLimit = 256

var (
	Segment_Checker = "segment_checker"
	Channel_Checker = "channel_checker"
	Balance_Checker = "balance_checker"
	Index_Checker   = "index_checker"
	CheckerOrder    = []string{Channel_Checker, Segment_Checker, Balance_Checker, Index_Checker}
)

type CheckerController struct {
	stopCh         chan struct{}
	manualCheckChs map[string]chan struct{}
	meta           *meta.Meta
	dist           *meta.DistributionManager
	targetMgr      *meta.TargetManager
	broker         meta.Broker
	nodeMgr        *session.NodeManager
	balancer       balance.Balance

	scheduler task.Scheduler
	checkers  map[string]Checker

	stopOnce sync.Once
}

func NewCheckerController(
	meta *meta.Meta,
	dist *meta.DistributionManager,
	targetMgr *meta.TargetManager,
	balancer balance.Balance,
	nodeMgr *session.NodeManager,
	scheduler task.Scheduler,
	broker meta.Broker,
) *CheckerController {
	// CheckerController runs checkers with the order,
	// the former checker has higher priority
	checkers := map[string]Checker{
		Channel_Checker: NewChannelChecker(meta, dist, targetMgr, balancer),
		Segment_Checker: NewSegmentChecker(meta, dist, targetMgr, balancer, nodeMgr),
		Balance_Checker: NewBalanceChecker(meta, balancer, nodeMgr, scheduler),
		Index_Checker:   NewIndexChecker(meta, dist, broker),
	}

	id := 0
	for _, checkerName := range CheckerOrder {
		checkers[checkerName].SetID(int64(id + 1))
	}

	manualCheckChs := map[string]chan struct{}{
		Channel_Checker: make(chan struct{}, 1),
		Segment_Checker: make(chan struct{}, 1),
		Balance_Checker: make(chan struct{}, 1),
	}

	return &CheckerController{
		stopCh:         make(chan struct{}),
		manualCheckChs: manualCheckChs,
		meta:           meta,
		dist:           dist,
		targetMgr:      targetMgr,
		scheduler:      scheduler,
		checkers:       checkers,
		broker:         broker,
	}
}

func (controller *CheckerController) Start(ctx context.Context) {
	for checkerType := range controller.checkers {
		go controller.StartChecker(ctx, checkerType)
	}
}

func getCheckerInterval(checkerType string) time.Duration {
	switch checkerType {
	case Segment_Checker:
		return Params.QueryCoordCfg.SegmentCheckInterval.GetAsDuration(time.Millisecond)
	case Channel_Checker:
		return Params.QueryCoordCfg.ChannelCheckInterval.GetAsDuration(time.Millisecond)
	case Balance_Checker:
		return Params.QueryCoordCfg.BalanceCheckInterval.GetAsDuration(time.Millisecond)
	case Index_Checker:
		return Params.QueryCoordCfg.IndexCheckInterval.GetAsDuration(time.Millisecond)
	default:
		return Params.QueryCoordCfg.CheckInterval.GetAsDuration(time.Millisecond)
	}
}

func (controller *CheckerController) StartChecker(ctx context.Context, checkerType string) {
	interval := getCheckerInterval(checkerType)
	ticker := time.NewTicker(interval)
	defer ticker.Stop()

	for {
		select {
		case <-ctx.Done():
			log.Info("Checker stopped due to context canceled",
				zap.String("type", checkerType))
			return

		case <-controller.stopCh:
			log.Info("Checker stopped",
				zap.String("type", checkerType))
			return

		case <-ticker.C:
			controller.check(ctx, checkerType)

		case <-controller.manualCheckChs[checkerType]:
			ticker.Stop()
			controller.check(ctx, checkerType)
			ticker.Reset(interval)
		}
	}
}

func (controller *CheckerController) Stop() {
	controller.stopOnce.Do(func() {
		close(controller.stopCh)
	})
}

func (controller *CheckerController) Check() {
	for _, checkCh := range controller.manualCheckChs {
		select {
		case checkCh <- struct{}{}:
		default:
		}
	}
}

// check is the real implementation of Check
func (controller *CheckerController) check(ctx context.Context, checkerType string) {
	checker := controller.checkers[checkerType]
	tasks := checker.Check(ctx)

	for _, task := range tasks {
		err := controller.scheduler.Add(task)
		if err != nil {
			task.Cancel(err)
			continue
		}
	}
}