mirror of
https://gitee.com/milvus-io/milvus.git
synced 2025-12-07 01:28:27 +08:00
122d024df4
issue: #43897, #44123 pr: #45266 also pick pr: #45237, #45264,#45244,#45275 fix: kafka should auto reset the offset from earliest to read (#45237) issue: #44172, #45210, #44851,#45244 kafka will auto reset the offset to "latest" if the offset is Out-of-range. the recovery of milvus wal cannot read any message from that. So once the offset is out-of-range, kafka should read from eariest to read the latest uncleared data. https://kafka.apache.org/documentation/#consumerconfigs_auto.offset.reset enhance: support alter collection/database with WAL-based DDL framework (#45266) issue: #43897 - Alter collection/database is implemented by WAL-based DDL framework now. - Support AlterCollection/AlterDatabase in wal now. - Alter operation can be synced by new CDC now. - Refactor some UT for alter DDL. fix: milvus role cannot stop at initializing state (#45244) issue: #45243 fix: support upgrading from 2.6.x -> 2.6.5 (#45264) issue: #43897 --------- Signed-off-by: chyezh <chyezh@outlook.com>
295 lines
8.4 KiB
Go
295 lines
8.4 KiB
Go
// Licensed to the LF AI & Data foundation under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package rootcoord
|
|
|
|
import (
|
|
"context"
|
|
"testing"
|
|
|
|
"github.com/cockroachdb/errors"
|
|
"github.com/stretchr/testify/assert"
|
|
"github.com/stretchr/testify/mock"
|
|
|
|
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
|
|
"github.com/milvus-io/milvus/internal/metastore/model"
|
|
"github.com/milvus-io/milvus/internal/mocks"
|
|
mockrootcoord "github.com/milvus-io/milvus/internal/rootcoord/mocks"
|
|
"github.com/milvus-io/milvus/pkg/v2/proto/datapb"
|
|
pb "github.com/milvus-io/milvus/pkg/v2/proto/etcdpb"
|
|
"github.com/milvus-io/milvus/pkg/v2/util/merr"
|
|
)
|
|
|
|
func TestServerBroker_ReleaseCollection(t *testing.T) {
|
|
t.Run("failed to execute", func(t *testing.T) {
|
|
c := newTestCore(withInvalidMixCoord())
|
|
b := newServerBroker(c)
|
|
ctx := context.Background()
|
|
err := b.ReleaseCollection(ctx, 1)
|
|
assert.Error(t, err)
|
|
})
|
|
|
|
t.Run("non success error code on execute", func(t *testing.T) {
|
|
c := newTestCore(withFailedMixCoord())
|
|
b := newServerBroker(c)
|
|
ctx := context.Background()
|
|
err := b.ReleaseCollection(ctx, 1)
|
|
assert.Error(t, err)
|
|
})
|
|
|
|
t.Run("success", func(t *testing.T) {
|
|
c := newTestCore(withValidMixCoord())
|
|
b := newServerBroker(c)
|
|
ctx := context.Background()
|
|
err := b.ReleaseCollection(ctx, 1)
|
|
assert.NoError(t, err)
|
|
})
|
|
}
|
|
|
|
func TestServerBroker_GetSegmentInfo(t *testing.T) {
|
|
t.Run("failed to execute", func(t *testing.T) {
|
|
c := newTestCore(withInvalidMixCoord())
|
|
b := newServerBroker(c)
|
|
ctx := context.Background()
|
|
_, err := b.GetQuerySegmentInfo(ctx, 1, []int64{1, 2})
|
|
assert.Error(t, err)
|
|
})
|
|
|
|
t.Run("non success error code on execute", func(t *testing.T) {
|
|
c := newTestCore(withFailedMixCoord())
|
|
b := newServerBroker(c)
|
|
ctx := context.Background()
|
|
resp, err := b.GetQuerySegmentInfo(ctx, 1, []int64{1, 2})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, commonpb.ErrorCode_UnexpectedError, resp.GetStatus().GetErrorCode())
|
|
})
|
|
|
|
t.Run("success", func(t *testing.T) {
|
|
c := newTestCore(withValidMixCoord())
|
|
b := newServerBroker(c)
|
|
ctx := context.Background()
|
|
resp, err := b.GetQuerySegmentInfo(ctx, 1, []int64{1, 2})
|
|
assert.NoError(t, err)
|
|
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
|
|
})
|
|
}
|
|
|
|
func TestServerBroker_WatchChannels(t *testing.T) {
|
|
t.Run("failed to execute", func(t *testing.T) {
|
|
defer cleanTestEnv()
|
|
|
|
c := newTestCore(withInvalidMixCoord(), withRocksMqTtSynchronizer())
|
|
b := newServerBroker(c)
|
|
ctx := context.Background()
|
|
err := b.WatchChannels(ctx, &watchInfo{})
|
|
assert.Error(t, err)
|
|
})
|
|
|
|
t.Run("non success error code on execute", func(t *testing.T) {
|
|
defer cleanTestEnv()
|
|
|
|
c := newTestCore(withFailedMixCoord(), withRocksMqTtSynchronizer())
|
|
b := newServerBroker(c)
|
|
ctx := context.Background()
|
|
err := b.WatchChannels(ctx, &watchInfo{})
|
|
assert.Error(t, err)
|
|
})
|
|
|
|
t.Run("success", func(t *testing.T) {
|
|
defer cleanTestEnv()
|
|
|
|
c := newTestCore(withValidMixCoord(), withRocksMqTtSynchronizer())
|
|
b := newServerBroker(c)
|
|
ctx := context.Background()
|
|
err := b.WatchChannels(ctx, &watchInfo{})
|
|
assert.NoError(t, err)
|
|
})
|
|
}
|
|
|
|
func TestServerBroker_UnwatchChannels(t *testing.T) {
|
|
// TODO: implement
|
|
b := newServerBroker(newTestCore())
|
|
ctx := context.Background()
|
|
b.UnwatchChannels(ctx, &watchInfo{})
|
|
}
|
|
|
|
func TestServerBroker_DropCollectionIndex(t *testing.T) {
|
|
t.Run("failed to execute", func(t *testing.T) {
|
|
c := newTestCore(withInvalidMixCoord())
|
|
b := newServerBroker(c)
|
|
ctx := context.Background()
|
|
err := b.DropCollectionIndex(ctx, 1, nil)
|
|
assert.Error(t, err)
|
|
})
|
|
|
|
t.Run("non success error code on execute", func(t *testing.T) {
|
|
c := newTestCore(withFailedMixCoord())
|
|
b := newServerBroker(c)
|
|
ctx := context.Background()
|
|
err := b.DropCollectionIndex(ctx, 1, nil)
|
|
assert.Error(t, err)
|
|
})
|
|
|
|
t.Run("success", func(t *testing.T) {
|
|
c := newTestCore(withValidMixCoord())
|
|
b := newServerBroker(c)
|
|
ctx := context.Background()
|
|
err := b.DropCollectionIndex(ctx, 1, nil)
|
|
assert.NoError(t, err)
|
|
})
|
|
}
|
|
|
|
func TestServerBroker_BroadcastAlteredCollection(t *testing.T) {
|
|
collMeta := &model.Collection{
|
|
CollectionID: 1,
|
|
StartPositions: []*commonpb.KeyDataPair{
|
|
{
|
|
Key: "0",
|
|
Data: []byte("0"),
|
|
},
|
|
},
|
|
Partitions: []*model.Partition{
|
|
{
|
|
PartitionID: 2,
|
|
PartitionName: "test_partition_name_1",
|
|
PartitionCreatedTimestamp: 0,
|
|
},
|
|
},
|
|
}
|
|
|
|
t.Run("get meta fail", func(t *testing.T) {
|
|
c := newTestCore(withInvalidMixCoord())
|
|
meta := mockrootcoord.NewIMetaTable(t)
|
|
meta.On("GetCollectionByID",
|
|
mock.Anything,
|
|
mock.Anything,
|
|
mock.Anything,
|
|
mock.Anything,
|
|
mock.Anything,
|
|
).Return(nil, errors.New("err"))
|
|
c.meta = meta
|
|
b := newServerBroker(c)
|
|
ctx := context.Background()
|
|
err := b.BroadcastAlteredCollection(ctx, 0)
|
|
assert.Error(t, err)
|
|
})
|
|
|
|
t.Run("failed to execute", func(t *testing.T) {
|
|
c := newTestCore(withInvalidMixCoord())
|
|
meta := mockrootcoord.NewIMetaTable(t)
|
|
meta.On("GetCollectionByID",
|
|
mock.Anything,
|
|
mock.Anything,
|
|
mock.Anything,
|
|
mock.Anything,
|
|
mock.Anything,
|
|
).Return(collMeta, nil)
|
|
mockGetDatabase(meta)
|
|
c.meta = meta
|
|
b := newServerBroker(c)
|
|
ctx := context.Background()
|
|
err := b.BroadcastAlteredCollection(ctx, 0)
|
|
assert.Error(t, err)
|
|
})
|
|
|
|
t.Run("non success error code on execute", func(t *testing.T) {
|
|
c := newTestCore(withFailedMixCoord())
|
|
meta := mockrootcoord.NewIMetaTable(t)
|
|
meta.On("GetCollectionByID",
|
|
mock.Anything,
|
|
mock.Anything,
|
|
mock.Anything,
|
|
mock.Anything,
|
|
mock.Anything,
|
|
).Return(collMeta, nil)
|
|
mockGetDatabase(meta)
|
|
c.meta = meta
|
|
b := newServerBroker(c)
|
|
ctx := context.Background()
|
|
err := b.BroadcastAlteredCollection(ctx, 0)
|
|
assert.Error(t, err)
|
|
})
|
|
|
|
t.Run("success", func(t *testing.T) {
|
|
c := newTestCore(withValidMixCoord())
|
|
meta := mockrootcoord.NewIMetaTable(t)
|
|
meta.On("GetCollectionByID",
|
|
mock.Anything,
|
|
mock.Anything,
|
|
mock.Anything,
|
|
mock.Anything,
|
|
mock.Anything,
|
|
).Return(collMeta, nil)
|
|
mockGetDatabase(meta)
|
|
c.meta = meta
|
|
b := newServerBroker(c)
|
|
ctx := context.Background()
|
|
err := b.BroadcastAlteredCollection(ctx, 1)
|
|
assert.NoError(t, err)
|
|
})
|
|
}
|
|
|
|
func TestServerBroker_GcConfirm(t *testing.T) {
|
|
t.Run("invalid datacoord", func(t *testing.T) {
|
|
dc := mocks.NewMixCoord(t)
|
|
dc.On("GcConfirm",
|
|
mock.Anything, // context.Context
|
|
mock.Anything, // *datapb.GcConfirmRequest
|
|
mock.Anything, // *datapb.GcConfirmRequest
|
|
).Return(nil, errors.New("error mock GcConfirm"))
|
|
c := newTestCore(withMixCoord(dc))
|
|
broker := newServerBroker(c)
|
|
assert.False(t, broker.GcConfirm(context.Background(), 100, 10000))
|
|
})
|
|
|
|
t.Run("non success", func(t *testing.T) {
|
|
dc := mocks.NewMixCoord(t)
|
|
err := errors.New("mock error")
|
|
dc.On("GcConfirm",
|
|
mock.Anything, // context.Context
|
|
mock.Anything, // *datapb.GcConfirmRequest
|
|
mock.Anything,
|
|
).Return(
|
|
&datapb.GcConfirmResponse{Status: merr.Status(err)},
|
|
nil)
|
|
c := newTestCore(withMixCoord(dc))
|
|
broker := newServerBroker(c)
|
|
assert.False(t, broker.GcConfirm(context.Background(), 100, 10000))
|
|
})
|
|
|
|
t.Run("normal case", func(t *testing.T) {
|
|
dc := mocks.NewMixCoord(t)
|
|
dc.On("GcConfirm",
|
|
mock.Anything, // context.Context
|
|
mock.Anything, // *datapb.GcConfirmRequest
|
|
mock.Anything,
|
|
).Return(
|
|
&datapb.GcConfirmResponse{Status: merr.Success(), GcFinished: true},
|
|
nil)
|
|
c := newTestCore(withMixCoord(dc))
|
|
broker := newServerBroker(c)
|
|
assert.True(t, broker.GcConfirm(context.Background(), 100, 10000))
|
|
})
|
|
}
|
|
|
|
func mockGetDatabase(meta *mockrootcoord.IMetaTable) {
|
|
db := model.NewDatabase(1, "default", pb.DatabaseState_DatabaseCreated, nil)
|
|
meta.EXPECT().GetDatabaseByName(mock.Anything, mock.Anything, mock.Anything).
|
|
Return(db, nil).Maybe()
|
|
meta.EXPECT().GetDatabaseByID(mock.Anything, mock.Anything, mock.Anything).
|
|
Return(db, nil).Maybe()
|
|
}
|