mirror of
https://gitee.com/milvus-io/milvus.git
synced 2025-12-06 09:08:43 +08:00
0c0630cc38
issue: #42942 This pr includes the following changes: 1. Added checks for index checker in querycoord to generate drop index tasks 2. Added drop index interface to querynode 3. To avoid search failure after dropping the index, the querynode allows the use of lazy mode (warmup=disable) to load raw data even when indexes contain raw data. 4. In segcore, loading the index no longer deletes raw data; instead, it evicts it. 5. In expr, the index is pinned to prevent concurrent errors. --------- Signed-off-by: sunby <sunbingyi1992@gmail.com>
111 lines
3.2 KiB
Go
111 lines
3.2 KiB
Go
package segcore
|
|
|
|
/*
|
|
#cgo pkg-config: milvus_core
|
|
|
|
#include "common/type_c.h"
|
|
#include "segcore/load_field_data_c.h"
|
|
*/
|
|
import "C"
|
|
|
|
import (
|
|
"unsafe"
|
|
|
|
"github.com/cockroachdb/errors"
|
|
|
|
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
|
|
"github.com/milvus-io/milvus/internal/storage"
|
|
"github.com/milvus-io/milvus/internal/util/initcore"
|
|
"github.com/milvus-io/milvus/pkg/v2/proto/datapb"
|
|
"github.com/milvus-io/milvus/pkg/v2/proto/segcorepb"
|
|
"github.com/milvus-io/milvus/pkg/v2/util/typeutil"
|
|
)
|
|
|
|
type RetrievePlanWithOffsets struct {
|
|
*RetrievePlan
|
|
Offsets []int64
|
|
}
|
|
|
|
type InsertRequest struct {
|
|
RowIDs []int64
|
|
Timestamps []typeutil.Timestamp
|
|
Record *segcorepb.InsertRecord
|
|
}
|
|
|
|
type DeleteRequest struct {
|
|
PrimaryKeys storage.PrimaryKeys
|
|
Timestamps []typeutil.Timestamp
|
|
}
|
|
|
|
type LoadFieldDataRequest struct {
|
|
Fields []LoadFieldDataInfo
|
|
MMapDir string
|
|
RowCount int64
|
|
StorageVersion int64
|
|
LoadPriority commonpb.LoadPriority
|
|
WarmupPolicy string
|
|
}
|
|
|
|
type LoadFieldDataInfo struct {
|
|
Field *datapb.FieldBinlog
|
|
EnableMMap bool
|
|
}
|
|
|
|
func (req *LoadFieldDataRequest) getCLoadFieldDataRequest() (result *cLoadFieldDataRequest, err error) {
|
|
var cLoadFieldDataInfo C.CLoadFieldDataInfo
|
|
status := C.NewLoadFieldDataInfo(&cLoadFieldDataInfo, C.int64_t(req.StorageVersion))
|
|
if err := ConsumeCStatusIntoError(&status); err != nil {
|
|
return nil, errors.Wrap(err, "NewLoadFieldDataInfo failed")
|
|
}
|
|
defer func() {
|
|
if err != nil {
|
|
C.DeleteLoadFieldDataInfo(cLoadFieldDataInfo)
|
|
}
|
|
}()
|
|
rowCount := C.int64_t(req.RowCount)
|
|
|
|
for _, field := range req.Fields {
|
|
cFieldID := C.int64_t(field.Field.GetFieldID())
|
|
status = C.AppendLoadFieldInfo(cLoadFieldDataInfo, cFieldID, rowCount)
|
|
if err := ConsumeCStatusIntoError(&status); err != nil {
|
|
return nil, errors.Wrapf(err, "AppendLoadFieldInfo failed at fieldID, %d", field.Field.GetFieldID())
|
|
}
|
|
for _, binlog := range field.Field.Binlogs {
|
|
cEntriesNum := C.int64_t(binlog.GetEntriesNum())
|
|
cMemorySize := C.int64_t(binlog.GetMemorySize())
|
|
cFile := C.CString(binlog.GetLogPath())
|
|
defer C.free(unsafe.Pointer(cFile))
|
|
|
|
status = C.AppendLoadFieldDataPath(cLoadFieldDataInfo, cFieldID, cEntriesNum, cMemorySize, cFile)
|
|
if err := ConsumeCStatusIntoError(&status); err != nil {
|
|
return nil, errors.Wrapf(err, "AppendLoadFieldDataPath failed at binlog, %d, %s", field.Field.GetFieldID(), binlog.GetLogPath())
|
|
}
|
|
}
|
|
|
|
C.EnableMmap(cLoadFieldDataInfo, cFieldID, C.bool(field.EnableMMap))
|
|
}
|
|
C.SetLoadPriority(cLoadFieldDataInfo, C.int32_t(req.LoadPriority))
|
|
if len(req.WarmupPolicy) > 0 {
|
|
warmupPolicy, err := initcore.ConvertCacheWarmupPolicy(req.WarmupPolicy)
|
|
if err != nil {
|
|
return nil, errors.Wrapf(err, "ConvertCacheWarmupPolicy failed at warmupPolicy, %s", req.WarmupPolicy)
|
|
}
|
|
C.AppendWarmupPolicy(cLoadFieldDataInfo, C.CacheWarmupPolicy(warmupPolicy))
|
|
}
|
|
return &cLoadFieldDataRequest{
|
|
cLoadFieldDataInfo: cLoadFieldDataInfo,
|
|
}, nil
|
|
}
|
|
|
|
type cLoadFieldDataRequest struct {
|
|
cLoadFieldDataInfo C.CLoadFieldDataInfo
|
|
}
|
|
|
|
func (req *cLoadFieldDataRequest) Release() {
|
|
C.DeleteLoadFieldDataInfo(req.cLoadFieldDataInfo)
|
|
}
|
|
|
|
type AddFieldDataInfoRequest = LoadFieldDataRequest
|
|
|
|
type AddFieldDataInfoResult struct{}
|