mirror of
https://gitee.com/milvus-io/milvus.git
synced 2026-01-07 19:31:51 +08:00
b18ebd9468
issue: https://github.com/milvus-io/milvus/issues/44123 <!-- This is an auto-generated comment: release notes by coderabbit.ai --> - Core invariant: legacy in-cluster CDC/replication plumbing (ReplicateMsg types, ReplicateID-based guards and flags) is obsolete — the system relies on standard msgstream positions, subPos/end-ts semantics and timetick ordering as the single source of truth for message ordering and skipping, so replication-specific channels/types/guards can be removed safely. - Removed/simplified logic (what and why): removed replication feature flags and params (ReplicateMsgChannel, TTMsgEnabled, CollectionReplicateEnable), ReplicateMsg type and its tests, ReplicateID constants/helpers and MergeProperties hooks, ReplicateConfig and its propagation (streamPipeline, StreamConfig, dispatcher, target), replicate-aware dispatcher/pipeline branches, and replicate-mode pre-checks/timestamp-allocation in proxy tasks — these implemented a redundant alternate “replicate-mode” pathway that duplicated position/end-ts and timetick logic. - Why this does NOT cause data loss or regression (concrete code paths): no persistence or core write paths were removed — proxy PreExecute flows (internal/proxy/task_*.go) still perform the same schema/ID/size validations and then follow the normal non-replicate execution path; dispatcher and pipeline continue to use position/subPos and pullback/end-ts in Seek/grouping (pkg/mq/msgdispatcher/dispatcher.go, internal/util/pipeline/stream_pipeline.go), so skipping and ordering behavior remains unchanged; timetick emission in rootcoord (sendMinDdlTsAsTt) is now ungated (no silent suppression), preserving or increasing timetick delivery rather than removing it. - PR type and net effect: Enhancement/Refactor — removes deprecated replication API surface (types, helpers, config, tests) and replication branches, simplifies public APIs and constructor signatures, and reduces surface area for future maintenance while keeping DML/DDL persistence, ordering, and seek semantics intact. <!-- end of auto-generated comment: release notes by coderabbit.ai --> --------- Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
397 lines
9.1 KiB
Go
397 lines
9.1 KiB
Go
package proxy
|
|
|
|
import (
|
|
"context"
|
|
|
|
"go.uber.org/zap"
|
|
|
|
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
|
|
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
|
|
"github.com/milvus-io/milvus/internal/types"
|
|
"github.com/milvus-io/milvus/pkg/v2/common"
|
|
"github.com/milvus-io/milvus/pkg/v2/log"
|
|
"github.com/milvus-io/milvus/pkg/v2/proto/rootcoordpb"
|
|
"github.com/milvus-io/milvus/pkg/v2/util/commonpbutil"
|
|
"github.com/milvus-io/milvus/pkg/v2/util/funcutil"
|
|
"github.com/milvus-io/milvus/pkg/v2/util/merr"
|
|
"github.com/milvus-io/milvus/pkg/v2/util/paramtable"
|
|
"github.com/milvus-io/milvus/pkg/v2/util/timestamptz"
|
|
)
|
|
|
|
type createDatabaseTask struct {
|
|
baseTask
|
|
Condition
|
|
*milvuspb.CreateDatabaseRequest
|
|
ctx context.Context
|
|
mixCoord types.MixCoordClient
|
|
result *commonpb.Status
|
|
}
|
|
|
|
func (cdt *createDatabaseTask) TraceCtx() context.Context {
|
|
return cdt.ctx
|
|
}
|
|
|
|
func (cdt *createDatabaseTask) ID() UniqueID {
|
|
return cdt.Base.MsgID
|
|
}
|
|
|
|
func (cdt *createDatabaseTask) SetID(uid UniqueID) {
|
|
cdt.Base.MsgID = uid
|
|
}
|
|
|
|
func (cdt *createDatabaseTask) Name() string {
|
|
return CreateDatabaseTaskName
|
|
}
|
|
|
|
func (cdt *createDatabaseTask) Type() commonpb.MsgType {
|
|
return cdt.Base.MsgType
|
|
}
|
|
|
|
func (cdt *createDatabaseTask) BeginTs() Timestamp {
|
|
return cdt.Base.Timestamp
|
|
}
|
|
|
|
func (cdt *createDatabaseTask) EndTs() Timestamp {
|
|
return cdt.Base.Timestamp
|
|
}
|
|
|
|
func (cdt *createDatabaseTask) SetTs(ts Timestamp) {
|
|
cdt.Base.Timestamp = ts
|
|
}
|
|
|
|
func (cdt *createDatabaseTask) OnEnqueue() error {
|
|
if cdt.Base == nil {
|
|
cdt.Base = commonpbutil.NewMsgBase()
|
|
}
|
|
cdt.Base.MsgType = commonpb.MsgType_CreateDatabase
|
|
cdt.Base.SourceID = paramtable.GetNodeID()
|
|
return nil
|
|
}
|
|
|
|
func (cdt *createDatabaseTask) PreExecute(ctx context.Context) error {
|
|
err := ValidateDatabaseName(cdt.GetDbName())
|
|
if err != nil {
|
|
return err
|
|
}
|
|
tz, exist := funcutil.TryGetAttrByKeyFromRepeatedKV(common.TimezoneKey, cdt.GetProperties())
|
|
if exist && !timestamptz.IsTimezoneValid(tz) {
|
|
return merr.WrapErrParameterInvalidMsg("unknown or invalid IANA Time Zone ID: %s", tz)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (cdt *createDatabaseTask) Execute(ctx context.Context) error {
|
|
var err error
|
|
cdt.result, err = cdt.mixCoord.CreateDatabase(ctx, cdt.CreateDatabaseRequest)
|
|
err = merr.CheckRPCCall(cdt.result, err)
|
|
return err
|
|
}
|
|
|
|
func (cdt *createDatabaseTask) PostExecute(ctx context.Context) error {
|
|
return nil
|
|
}
|
|
|
|
type dropDatabaseTask struct {
|
|
baseTask
|
|
Condition
|
|
*milvuspb.DropDatabaseRequest
|
|
ctx context.Context
|
|
mixCoord types.MixCoordClient
|
|
result *commonpb.Status
|
|
}
|
|
|
|
func (ddt *dropDatabaseTask) TraceCtx() context.Context {
|
|
return ddt.ctx
|
|
}
|
|
|
|
func (ddt *dropDatabaseTask) ID() UniqueID {
|
|
return ddt.Base.MsgID
|
|
}
|
|
|
|
func (ddt *dropDatabaseTask) SetID(uid UniqueID) {
|
|
ddt.Base.MsgID = uid
|
|
}
|
|
|
|
func (ddt *dropDatabaseTask) Name() string {
|
|
return DropCollectionTaskName
|
|
}
|
|
|
|
func (ddt *dropDatabaseTask) Type() commonpb.MsgType {
|
|
return ddt.Base.MsgType
|
|
}
|
|
|
|
func (ddt *dropDatabaseTask) BeginTs() Timestamp {
|
|
return ddt.Base.Timestamp
|
|
}
|
|
|
|
func (ddt *dropDatabaseTask) EndTs() Timestamp {
|
|
return ddt.Base.Timestamp
|
|
}
|
|
|
|
func (ddt *dropDatabaseTask) SetTs(ts Timestamp) {
|
|
ddt.Base.Timestamp = ts
|
|
}
|
|
|
|
func (ddt *dropDatabaseTask) OnEnqueue() error {
|
|
if ddt.Base == nil {
|
|
ddt.Base = commonpbutil.NewMsgBase()
|
|
}
|
|
ddt.Base.MsgType = commonpb.MsgType_DropDatabase
|
|
ddt.Base.SourceID = paramtable.GetNodeID()
|
|
return nil
|
|
}
|
|
|
|
func (ddt *dropDatabaseTask) PreExecute(ctx context.Context) error {
|
|
return ValidateDatabaseName(ddt.GetDbName())
|
|
}
|
|
|
|
func (ddt *dropDatabaseTask) Execute(ctx context.Context) error {
|
|
var err error
|
|
ddt.result, err = ddt.mixCoord.DropDatabase(ctx, ddt.DropDatabaseRequest)
|
|
|
|
err = merr.CheckRPCCall(ddt.result, err)
|
|
if err == nil {
|
|
globalMetaCache.RemoveDatabase(ctx, ddt.DbName)
|
|
}
|
|
return err
|
|
}
|
|
|
|
func (ddt *dropDatabaseTask) PostExecute(ctx context.Context) error {
|
|
return nil
|
|
}
|
|
|
|
type listDatabaseTask struct {
|
|
baseTask
|
|
Condition
|
|
*milvuspb.ListDatabasesRequest
|
|
ctx context.Context
|
|
mixCoord types.MixCoordClient
|
|
result *milvuspb.ListDatabasesResponse
|
|
}
|
|
|
|
func (ldt *listDatabaseTask) TraceCtx() context.Context {
|
|
return ldt.ctx
|
|
}
|
|
|
|
func (ldt *listDatabaseTask) ID() UniqueID {
|
|
return ldt.Base.MsgID
|
|
}
|
|
|
|
func (ldt *listDatabaseTask) SetID(uid UniqueID) {
|
|
ldt.Base.MsgID = uid
|
|
}
|
|
|
|
func (ldt *listDatabaseTask) Name() string {
|
|
return ListDatabaseTaskName
|
|
}
|
|
|
|
func (ldt *listDatabaseTask) Type() commonpb.MsgType {
|
|
return ldt.Base.MsgType
|
|
}
|
|
|
|
func (ldt *listDatabaseTask) BeginTs() Timestamp {
|
|
return ldt.Base.Timestamp
|
|
}
|
|
|
|
func (ldt *listDatabaseTask) EndTs() Timestamp {
|
|
return ldt.Base.Timestamp
|
|
}
|
|
|
|
func (ldt *listDatabaseTask) SetTs(ts Timestamp) {
|
|
ldt.Base.Timestamp = ts
|
|
}
|
|
|
|
func (ldt *listDatabaseTask) OnEnqueue() error {
|
|
ldt.Base = commonpbutil.NewMsgBase()
|
|
ldt.Base.MsgType = commonpb.MsgType_ListDatabases
|
|
ldt.Base.SourceID = paramtable.GetNodeID()
|
|
return nil
|
|
}
|
|
|
|
func (ldt *listDatabaseTask) PreExecute(ctx context.Context) error {
|
|
return nil
|
|
}
|
|
|
|
func (ldt *listDatabaseTask) Execute(ctx context.Context) error {
|
|
var err error
|
|
ctx = AppendUserInfoForRPC(ctx)
|
|
ldt.result, err = ldt.mixCoord.ListDatabases(ctx, ldt.ListDatabasesRequest)
|
|
return merr.CheckRPCCall(ldt.result, err)
|
|
}
|
|
|
|
func (ldt *listDatabaseTask) PostExecute(ctx context.Context) error {
|
|
return nil
|
|
}
|
|
|
|
type alterDatabaseTask struct {
|
|
baseTask
|
|
Condition
|
|
*milvuspb.AlterDatabaseRequest
|
|
ctx context.Context
|
|
mixCoord types.MixCoordClient
|
|
result *commonpb.Status
|
|
}
|
|
|
|
func (t *alterDatabaseTask) TraceCtx() context.Context {
|
|
return t.ctx
|
|
}
|
|
|
|
func (t *alterDatabaseTask) ID() UniqueID {
|
|
return t.Base.MsgID
|
|
}
|
|
|
|
func (t *alterDatabaseTask) SetID(uid UniqueID) {
|
|
t.Base.MsgID = uid
|
|
}
|
|
|
|
func (t *alterDatabaseTask) Name() string {
|
|
return AlterDatabaseTaskName
|
|
}
|
|
|
|
func (t *alterDatabaseTask) Type() commonpb.MsgType {
|
|
return t.Base.MsgType
|
|
}
|
|
|
|
func (t *alterDatabaseTask) BeginTs() Timestamp {
|
|
return t.Base.Timestamp
|
|
}
|
|
|
|
func (t *alterDatabaseTask) EndTs() Timestamp {
|
|
return t.Base.Timestamp
|
|
}
|
|
|
|
func (t *alterDatabaseTask) SetTs(ts Timestamp) {
|
|
t.Base.Timestamp = ts
|
|
}
|
|
|
|
func (t *alterDatabaseTask) OnEnqueue() error {
|
|
if t.Base == nil {
|
|
t.Base = commonpbutil.NewMsgBase()
|
|
}
|
|
t.Base.MsgType = commonpb.MsgType_AlterDatabase
|
|
t.Base.SourceID = paramtable.GetNodeID()
|
|
return nil
|
|
}
|
|
|
|
func (t *alterDatabaseTask) PreExecute(ctx context.Context) error {
|
|
if len(t.GetProperties()) > 0 {
|
|
// Check the validation of timezone
|
|
userDefinedTimezone, exist := funcutil.TryGetAttrByKeyFromRepeatedKV(common.TimezoneKey, t.Properties)
|
|
if exist && !timestamptz.IsTimezoneValid(userDefinedTimezone) {
|
|
return merr.WrapErrParameterInvalidMsg("unknown or invalid IANA Time Zone ID: %s", userDefinedTimezone)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (t *alterDatabaseTask) Execute(ctx context.Context) error {
|
|
var err error
|
|
|
|
req := &rootcoordpb.AlterDatabaseRequest{
|
|
Base: t.AlterDatabaseRequest.GetBase(),
|
|
DbName: t.AlterDatabaseRequest.GetDbName(),
|
|
DbId: t.AlterDatabaseRequest.GetDbId(),
|
|
Properties: t.AlterDatabaseRequest.GetProperties(),
|
|
DeleteKeys: t.AlterDatabaseRequest.GetDeleteKeys(),
|
|
}
|
|
|
|
ret, err := t.mixCoord.AlterDatabase(ctx, req)
|
|
err = merr.CheckRPCCall(ret, err)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
t.result = ret
|
|
return nil
|
|
}
|
|
|
|
func (t *alterDatabaseTask) PostExecute(ctx context.Context) error {
|
|
return nil
|
|
}
|
|
|
|
type describeDatabaseTask struct {
|
|
baseTask
|
|
Condition
|
|
*milvuspb.DescribeDatabaseRequest
|
|
ctx context.Context
|
|
mixCoord types.MixCoordClient
|
|
result *milvuspb.DescribeDatabaseResponse
|
|
}
|
|
|
|
func (t *describeDatabaseTask) TraceCtx() context.Context {
|
|
return t.ctx
|
|
}
|
|
|
|
func (t *describeDatabaseTask) ID() UniqueID {
|
|
return t.Base.MsgID
|
|
}
|
|
|
|
func (t *describeDatabaseTask) SetID(uid UniqueID) {
|
|
t.Base.MsgID = uid
|
|
}
|
|
|
|
func (t *describeDatabaseTask) Name() string {
|
|
return AlterDatabaseTaskName
|
|
}
|
|
|
|
func (t *describeDatabaseTask) Type() commonpb.MsgType {
|
|
return t.Base.MsgType
|
|
}
|
|
|
|
func (t *describeDatabaseTask) BeginTs() Timestamp {
|
|
return t.Base.Timestamp
|
|
}
|
|
|
|
func (t *describeDatabaseTask) EndTs() Timestamp {
|
|
return t.Base.Timestamp
|
|
}
|
|
|
|
func (t *describeDatabaseTask) SetTs(ts Timestamp) {
|
|
t.Base.Timestamp = ts
|
|
}
|
|
|
|
func (t *describeDatabaseTask) OnEnqueue() error {
|
|
if t.Base == nil {
|
|
t.Base = commonpbutil.NewMsgBase()
|
|
}
|
|
t.Base.MsgType = commonpb.MsgType_DescribeDatabase
|
|
t.Base.SourceID = paramtable.GetNodeID()
|
|
return nil
|
|
}
|
|
|
|
func (t *describeDatabaseTask) PreExecute(ctx context.Context) error {
|
|
return nil
|
|
}
|
|
|
|
func (t *describeDatabaseTask) Execute(ctx context.Context) error {
|
|
req := &rootcoordpb.DescribeDatabaseRequest{
|
|
Base: t.DescribeDatabaseRequest.GetBase(),
|
|
DbName: t.DescribeDatabaseRequest.GetDbName(),
|
|
}
|
|
|
|
ctx = AppendUserInfoForRPC(ctx)
|
|
ret, err := t.mixCoord.DescribeDatabase(ctx, req)
|
|
if err != nil {
|
|
log.Ctx(ctx).Warn("DescribeDatabase failed", zap.Error(err))
|
|
return err
|
|
}
|
|
|
|
if err := merr.CheckRPCCall(ret, err); err != nil {
|
|
log.Ctx(ctx).Warn("DescribeDatabase failed", zap.Error(err))
|
|
return err
|
|
}
|
|
|
|
t.result = &milvuspb.DescribeDatabaseResponse{
|
|
Status: ret.GetStatus(),
|
|
DbName: ret.GetDbName(),
|
|
DbID: ret.GetDbID(),
|
|
CreatedTimestamp: ret.GetCreatedTimestamp(),
|
|
Properties: ret.GetProperties(),
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (t *describeDatabaseTask) PostExecute(ctx context.Context) error {
|
|
return nil
|
|
}
|