mirror of
https://gitee.com/milvus-io/milvus.git
synced 2025-12-06 17:18:35 +08:00
enhance: forbid build analyzer at proxy (#44067)
relate: https://github.com/milvus-io/milvus/issues/43687 We used to run the temporary analyzer and validate analyzer on the proxy, but the proxy should not be a computation-heavy node. This PR move all analyzer calculations to the streaming node. --------- Signed-off-by: aoiasd <zhicheng.yue@zilliz.com>
This commit is contained in:
parent
20dcb45b3d
commit
cfeb095ad7
@ -478,7 +478,7 @@ func (mr *MilvusRoles) Run() {
|
||||
|
||||
exp, err := tracer.CreateTracerExporter(params)
|
||||
if err != nil {
|
||||
log.Warn("Init tracer faield", zap.Error(err))
|
||||
log.Warn("Init tracer failed", zap.Error(err))
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@ -1193,6 +1193,14 @@ func (s *mixCoordImpl) ListLoadedSegments(ctx context.Context, req *querypb.List
|
||||
return s.queryCoordServer.ListLoadedSegments(ctx, req)
|
||||
}
|
||||
|
||||
func (s *mixCoordImpl) RunAnalyzer(ctx context.Context, req *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) {
|
||||
return s.queryCoordServer.RunAnalyzer(ctx, req)
|
||||
}
|
||||
|
||||
func (s *mixCoordImpl) ValidateAnalyzer(ctx context.Context, req *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error) {
|
||||
return s.queryCoordServer.ValidateAnalyzer(ctx, req)
|
||||
}
|
||||
|
||||
func (s *mixCoordImpl) FlushAll(ctx context.Context, req *datapb.FlushAllRequest) (*datapb.FlushAllResponse, error) {
|
||||
return s.datacoordServer.FlushAll(ctx, req)
|
||||
}
|
||||
|
||||
@ -922,6 +922,14 @@ func (s *mockMixCoord) ListFileResources(ctx context.Context, req *milvuspb.List
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (s *mockMixCoord) RunAnalyzer(ctx context.Context, req *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
func (s *mockMixCoord) ValidateAnalyzer(ctx context.Context, req *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error) {
|
||||
panic("implement me")
|
||||
}
|
||||
|
||||
type mockHandler struct {
|
||||
meta *meta
|
||||
}
|
||||
|
||||
@ -1893,3 +1893,25 @@ func (c *Client) ListFileResources(ctx context.Context, req *milvuspb.ListFileRe
|
||||
return client.ListFileResources(ctx, req)
|
||||
})
|
||||
}
|
||||
|
||||
func (c *Client) RunAnalyzer(ctx context.Context, req *querypb.RunAnalyzerRequest, opts ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error) {
|
||||
req = typeutil.Clone(req)
|
||||
commonpbutil.UpdateMsgBase(
|
||||
req.GetBase(),
|
||||
commonpbutil.FillMsgBaseFromClient(paramtable.GetNodeID(), commonpbutil.WithTargetID(c.grpcClient.GetNodeID())),
|
||||
)
|
||||
return wrapGrpcCall(ctx, c, func(client MixCoordClient) (*milvuspb.RunAnalyzerResponse, error) {
|
||||
return client.RunAnalyzer(ctx, req)
|
||||
})
|
||||
}
|
||||
|
||||
func (c *Client) ValidateAnalyzer(ctx context.Context, req *querypb.ValidateAnalyzerRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
|
||||
req = typeutil.Clone(req)
|
||||
commonpbutil.UpdateMsgBase(
|
||||
req.GetBase(),
|
||||
commonpbutil.FillMsgBaseFromClient(paramtable.GetNodeID(), commonpbutil.WithTargetID(c.grpcClient.GetNodeID())),
|
||||
)
|
||||
return wrapGrpcCall(ctx, c, func(client MixCoordClient) (*commonpb.Status, error) {
|
||||
return client.ValidateAnalyzer(ctx, req)
|
||||
})
|
||||
}
|
||||
|
||||
@ -384,6 +384,12 @@ func Test_NewClient(t *testing.T) {
|
||||
|
||||
r40, err := client.CheckBalanceStatus(ctx, nil)
|
||||
retCheck(retNotNil, r40, err)
|
||||
|
||||
r41, err := client.RunAnalyzer(ctx, nil)
|
||||
retCheck(retNotNil, r41, err)
|
||||
|
||||
r42, err := client.ValidateAnalyzer(ctx, nil)
|
||||
retCheck(retNotNil, r42, err)
|
||||
}
|
||||
|
||||
client.(*Client).grpcClient = &mock.GRPCClientBase[MixCoordClient]{
|
||||
|
||||
@ -920,6 +920,14 @@ func (s *Server) ListLoadedSegments(ctx context.Context, req *querypb.ListLoaded
|
||||
return s.mixCoord.ListLoadedSegments(ctx, req)
|
||||
}
|
||||
|
||||
func (s *Server) RunAnalyzer(ctx context.Context, req *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) {
|
||||
return s.mixCoord.RunAnalyzer(ctx, req)
|
||||
}
|
||||
|
||||
func (s *Server) ValidateAnalyzer(ctx context.Context, req *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error) {
|
||||
return s.mixCoord.ValidateAnalyzer(ctx, req)
|
||||
}
|
||||
|
||||
// AddFileResource add file resource
|
||||
func (s *Server) AddFileResource(ctx context.Context, req *milvuspb.AddFileResourceRequest) (*commonpb.Status, error) {
|
||||
return s.mixCoord.AddFileResource(ctx, req)
|
||||
|
||||
@ -760,4 +760,22 @@ func Test_NewServer(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
|
||||
})
|
||||
|
||||
t.Run("RunAnalyzer", func(t *testing.T) {
|
||||
req := &querypb.RunAnalyzerRequest{}
|
||||
mockMixCoord.EXPECT().RunAnalyzer(mock.Anything, req).Return(&milvuspb.RunAnalyzerResponse{
|
||||
Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success},
|
||||
}, nil)
|
||||
resp, err := server.RunAnalyzer(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
|
||||
})
|
||||
|
||||
t.Run("ValidateAnalyzer", func(t *testing.T) {
|
||||
req := &querypb.ValidateAnalyzerRequest{}
|
||||
mockMixCoord.EXPECT().ValidateAnalyzer(mock.Anything, req).Return(&commonpb.Status{ErrorCode: commonpb.ErrorCode_Success}, nil)
|
||||
resp, err := server.ValidateAnalyzer(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, commonpb.ErrorCode_Success, resp.ErrorCode)
|
||||
})
|
||||
}
|
||||
|
||||
@ -688,6 +688,14 @@ func Test_NewServer(t *testing.T) {
|
||||
assert.Nil(t, err)
|
||||
})
|
||||
|
||||
t.Run("RunAnalyzer", func(t *testing.T) {
|
||||
mockProxy.EXPECT().RunAnalyzer(mock.Anything, mock.Anything).Return(&milvuspb.RunAnalyzerResponse{
|
||||
Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success},
|
||||
}, nil)
|
||||
_, err := server.RunAnalyzer(ctx, &milvuspb.RunAnalyzerRequest{})
|
||||
assert.NoError(t, err)
|
||||
})
|
||||
|
||||
t.Run("Run with different config", func(t *testing.T) {
|
||||
mockProxy.EXPECT().Init().Return(nil)
|
||||
mockProxy.EXPECT().Start().Return(nil)
|
||||
|
||||
@ -394,3 +394,14 @@ func (c *Client) DropIndex(ctx context.Context, req *querypb.DropIndexRequest, _
|
||||
return client.DropIndex(ctx, req)
|
||||
})
|
||||
}
|
||||
|
||||
func (c *Client) ValidateAnalyzer(ctx context.Context, req *querypb.ValidateAnalyzerRequest, _ ...grpc.CallOption) (*commonpb.Status, error) {
|
||||
req = typeutil.Clone(req)
|
||||
commonpbutil.UpdateMsgBase(
|
||||
req.GetBase(),
|
||||
commonpbutil.FillMsgBaseFromClient(c.nodeID),
|
||||
)
|
||||
return wrapGrpcCall(ctx, c, func(client querypb.QueryNodeClient) (*commonpb.Status, error) {
|
||||
return client.ValidateAnalyzer(ctx, req)
|
||||
})
|
||||
}
|
||||
|
||||
@ -111,6 +111,12 @@ func Test_NewClient(t *testing.T) {
|
||||
r21, err := client.DeleteBatch(ctx, nil)
|
||||
retCheck(retNotNil, r21, err)
|
||||
|
||||
r22, err := client.RunAnalyzer(ctx, nil)
|
||||
retCheck(retNotNil, r22, err)
|
||||
|
||||
r23, err := client.ValidateAnalyzer(ctx, nil)
|
||||
retCheck(retNotNil, r23, err)
|
||||
|
||||
// stream rpc
|
||||
client, err := client.QueryStream(ctx, nil)
|
||||
retCheck(retNotNil, client, err)
|
||||
|
||||
@ -406,3 +406,7 @@ func (s *Server) RunAnalyzer(ctx context.Context, req *querypb.RunAnalyzerReques
|
||||
func (s *Server) DropIndex(ctx context.Context, req *querypb.DropIndexRequest) (*commonpb.Status, error) {
|
||||
return s.querynode.DropIndex(ctx, req)
|
||||
}
|
||||
|
||||
func (s *Server) ValidateAnalyzer(ctx context.Context, req *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error) {
|
||||
return s.querynode.ValidateAnalyzer(ctx, req)
|
||||
}
|
||||
|
||||
@ -280,6 +280,24 @@ func Test_NewServer(t *testing.T) {
|
||||
assert.NoError(t, merr.CheckRPCCall(resp, err))
|
||||
})
|
||||
|
||||
t.Run("RunAnalyzer", func(t *testing.T) {
|
||||
mockQN.EXPECT().RunAnalyzer(mock.Anything, mock.Anything).Return(&milvuspb.RunAnalyzerResponse{
|
||||
Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success},
|
||||
}, nil)
|
||||
req := &querypb.RunAnalyzerRequest{}
|
||||
resp, err := server.RunAnalyzer(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
|
||||
})
|
||||
|
||||
t.Run("ValidateAnalyzer", func(t *testing.T) {
|
||||
mockQN.EXPECT().ValidateAnalyzer(mock.Anything, mock.Anything).Return(&commonpb.Status{ErrorCode: commonpb.ErrorCode_Success}, nil)
|
||||
req := &querypb.ValidateAnalyzerRequest{}
|
||||
resp, err := server.ValidateAnalyzer(ctx, req)
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, commonpb.ErrorCode_Success, resp.ErrorCode)
|
||||
})
|
||||
|
||||
err = server.Stop()
|
||||
assert.NoError(t, err)
|
||||
}
|
||||
|
||||
@ -633,7 +633,7 @@ func (ss *SuffixSnapshot) startBackgroundGC(ctx context.Context) {
|
||||
|
||||
func (ss *SuffixSnapshot) getOriginalKey(snapshotKey string) (string, error) {
|
||||
if !strings.HasPrefix(snapshotKey, ss.snapshotPrefix) {
|
||||
return "", fmt.Errorf("get original key failed, invailed snapshot key:%s", snapshotKey)
|
||||
return "", fmt.Errorf("get original key failed, invaild snapshot key:%s", snapshotKey)
|
||||
}
|
||||
// collect keys that parent node is snapshot node if the corresponding the latest ts is expired.
|
||||
idx := strings.LastIndex(snapshotKey, ss.separator)
|
||||
|
||||
@ -6973,6 +6973,65 @@ func (_c *MixCoord_ResumeNode_Call) RunAndReturn(run func(context.Context, *quer
|
||||
return _c
|
||||
}
|
||||
|
||||
// RunAnalyzer provides a mock function with given fields: _a0, _a1
|
||||
func (_m *MixCoord) RunAnalyzer(_a0 context.Context, _a1 *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) {
|
||||
ret := _m.Called(_a0, _a1)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for RunAnalyzer")
|
||||
}
|
||||
|
||||
var r0 *milvuspb.RunAnalyzerResponse
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error)); ok {
|
||||
return rf(_a0, _a1)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *querypb.RunAnalyzerRequest) *milvuspb.RunAnalyzerResponse); ok {
|
||||
r0 = rf(_a0, _a1)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*milvuspb.RunAnalyzerResponse)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *querypb.RunAnalyzerRequest) error); ok {
|
||||
r1 = rf(_a0, _a1)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MixCoord_RunAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RunAnalyzer'
|
||||
type MixCoord_RunAnalyzer_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// RunAnalyzer is a helper method to define mock.On call
|
||||
// - _a0 context.Context
|
||||
// - _a1 *querypb.RunAnalyzerRequest
|
||||
func (_e *MixCoord_Expecter) RunAnalyzer(_a0 interface{}, _a1 interface{}) *MixCoord_RunAnalyzer_Call {
|
||||
return &MixCoord_RunAnalyzer_Call{Call: _e.mock.On("RunAnalyzer", _a0, _a1)}
|
||||
}
|
||||
|
||||
func (_c *MixCoord_RunAnalyzer_Call) Run(run func(_a0 context.Context, _a1 *querypb.RunAnalyzerRequest)) *MixCoord_RunAnalyzer_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(*querypb.RunAnalyzerRequest))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MixCoord_RunAnalyzer_Call) Return(_a0 *milvuspb.RunAnalyzerResponse, _a1 error) *MixCoord_RunAnalyzer_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MixCoord_RunAnalyzer_Call) RunAndReturn(run func(context.Context, *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error)) *MixCoord_RunAnalyzer_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// SaveBinlogPaths provides a mock function with given fields: _a0, _a1
|
||||
func (_m *MixCoord) SaveBinlogPaths(_a0 context.Context, _a1 *datapb.SaveBinlogPathsRequest) (*commonpb.Status, error) {
|
||||
ret := _m.Called(_a0, _a1)
|
||||
@ -8762,6 +8821,65 @@ func (_c *MixCoord_UpdateStateCode_Call) RunAndReturn(run func(commonpb.StateCod
|
||||
return _c
|
||||
}
|
||||
|
||||
// ValidateAnalyzer provides a mock function with given fields: _a0, _a1
|
||||
func (_m *MixCoord) ValidateAnalyzer(_a0 context.Context, _a1 *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error) {
|
||||
ret := _m.Called(_a0, _a1)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for ValidateAnalyzer")
|
||||
}
|
||||
|
||||
var r0 *commonpb.Status
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error)); ok {
|
||||
return rf(_a0, _a1)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest) *commonpb.Status); ok {
|
||||
r0 = rf(_a0, _a1)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*commonpb.Status)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *querypb.ValidateAnalyzerRequest) error); ok {
|
||||
r1 = rf(_a0, _a1)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MixCoord_ValidateAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ValidateAnalyzer'
|
||||
type MixCoord_ValidateAnalyzer_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// ValidateAnalyzer is a helper method to define mock.On call
|
||||
// - _a0 context.Context
|
||||
// - _a1 *querypb.ValidateAnalyzerRequest
|
||||
func (_e *MixCoord_Expecter) ValidateAnalyzer(_a0 interface{}, _a1 interface{}) *MixCoord_ValidateAnalyzer_Call {
|
||||
return &MixCoord_ValidateAnalyzer_Call{Call: _e.mock.On("ValidateAnalyzer", _a0, _a1)}
|
||||
}
|
||||
|
||||
func (_c *MixCoord_ValidateAnalyzer_Call) Run(run func(_a0 context.Context, _a1 *querypb.ValidateAnalyzerRequest)) *MixCoord_ValidateAnalyzer_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(*querypb.ValidateAnalyzerRequest))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MixCoord_ValidateAnalyzer_Call) Return(_a0 *commonpb.Status, _a1 error) *MixCoord_ValidateAnalyzer_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MixCoord_ValidateAnalyzer_Call) RunAndReturn(run func(context.Context, *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error)) *MixCoord_ValidateAnalyzer_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// WatchChannels provides a mock function with given fields: _a0, _a1
|
||||
func (_m *MixCoord) WatchChannels(_a0 context.Context, _a1 *datapb.WatchChannelsRequest) (*datapb.WatchChannelsResponse, error) {
|
||||
ret := _m.Called(_a0, _a1)
|
||||
|
||||
@ -8372,6 +8372,80 @@ func (_c *MockMixCoordClient_ResumeNode_Call) RunAndReturn(run func(context.Cont
|
||||
return _c
|
||||
}
|
||||
|
||||
// RunAnalyzer provides a mock function with given fields: ctx, in, opts
|
||||
func (_m *MockMixCoordClient) RunAnalyzer(ctx context.Context, in *querypb.RunAnalyzerRequest, opts ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error) {
|
||||
_va := make([]interface{}, len(opts))
|
||||
for _i := range opts {
|
||||
_va[_i] = opts[_i]
|
||||
}
|
||||
var _ca []interface{}
|
||||
_ca = append(_ca, ctx, in)
|
||||
_ca = append(_ca, _va...)
|
||||
ret := _m.Called(_ca...)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for RunAnalyzer")
|
||||
}
|
||||
|
||||
var r0 *milvuspb.RunAnalyzerResponse
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *querypb.RunAnalyzerRequest, ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error)); ok {
|
||||
return rf(ctx, in, opts...)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *querypb.RunAnalyzerRequest, ...grpc.CallOption) *milvuspb.RunAnalyzerResponse); ok {
|
||||
r0 = rf(ctx, in, opts...)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*milvuspb.RunAnalyzerResponse)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *querypb.RunAnalyzerRequest, ...grpc.CallOption) error); ok {
|
||||
r1 = rf(ctx, in, opts...)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockMixCoordClient_RunAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RunAnalyzer'
|
||||
type MockMixCoordClient_RunAnalyzer_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// RunAnalyzer is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - in *querypb.RunAnalyzerRequest
|
||||
// - opts ...grpc.CallOption
|
||||
func (_e *MockMixCoordClient_Expecter) RunAnalyzer(ctx interface{}, in interface{}, opts ...interface{}) *MockMixCoordClient_RunAnalyzer_Call {
|
||||
return &MockMixCoordClient_RunAnalyzer_Call{Call: _e.mock.On("RunAnalyzer",
|
||||
append([]interface{}{ctx, in}, opts...)...)}
|
||||
}
|
||||
|
||||
func (_c *MockMixCoordClient_RunAnalyzer_Call) Run(run func(ctx context.Context, in *querypb.RunAnalyzerRequest, opts ...grpc.CallOption)) *MockMixCoordClient_RunAnalyzer_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
variadicArgs := make([]grpc.CallOption, len(args)-2)
|
||||
for i, a := range args[2:] {
|
||||
if a != nil {
|
||||
variadicArgs[i] = a.(grpc.CallOption)
|
||||
}
|
||||
}
|
||||
run(args[0].(context.Context), args[1].(*querypb.RunAnalyzerRequest), variadicArgs...)
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMixCoordClient_RunAnalyzer_Call) Return(_a0 *milvuspb.RunAnalyzerResponse, _a1 error) *MockMixCoordClient_RunAnalyzer_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMixCoordClient_RunAnalyzer_Call) RunAndReturn(run func(context.Context, *querypb.RunAnalyzerRequest, ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error)) *MockMixCoordClient_RunAnalyzer_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// SaveBinlogPaths provides a mock function with given fields: ctx, in, opts
|
||||
func (_m *MockMixCoordClient) SaveBinlogPaths(ctx context.Context, in *datapb.SaveBinlogPathsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
|
||||
_va := make([]interface{}, len(opts))
|
||||
@ -10296,6 +10370,80 @@ func (_c *MockMixCoordClient_UpdateSegmentStatistics_Call) RunAndReturn(run func
|
||||
return _c
|
||||
}
|
||||
|
||||
// ValidateAnalyzer provides a mock function with given fields: ctx, in, opts
|
||||
func (_m *MockMixCoordClient) ValidateAnalyzer(ctx context.Context, in *querypb.ValidateAnalyzerRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
|
||||
_va := make([]interface{}, len(opts))
|
||||
for _i := range opts {
|
||||
_va[_i] = opts[_i]
|
||||
}
|
||||
var _ca []interface{}
|
||||
_ca = append(_ca, ctx, in)
|
||||
_ca = append(_ca, _va...)
|
||||
ret := _m.Called(_ca...)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for ValidateAnalyzer")
|
||||
}
|
||||
|
||||
var r0 *commonpb.Status
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest, ...grpc.CallOption) (*commonpb.Status, error)); ok {
|
||||
return rf(ctx, in, opts...)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest, ...grpc.CallOption) *commonpb.Status); ok {
|
||||
r0 = rf(ctx, in, opts...)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*commonpb.Status)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *querypb.ValidateAnalyzerRequest, ...grpc.CallOption) error); ok {
|
||||
r1 = rf(ctx, in, opts...)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockMixCoordClient_ValidateAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ValidateAnalyzer'
|
||||
type MockMixCoordClient_ValidateAnalyzer_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// ValidateAnalyzer is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - in *querypb.ValidateAnalyzerRequest
|
||||
// - opts ...grpc.CallOption
|
||||
func (_e *MockMixCoordClient_Expecter) ValidateAnalyzer(ctx interface{}, in interface{}, opts ...interface{}) *MockMixCoordClient_ValidateAnalyzer_Call {
|
||||
return &MockMixCoordClient_ValidateAnalyzer_Call{Call: _e.mock.On("ValidateAnalyzer",
|
||||
append([]interface{}{ctx, in}, opts...)...)}
|
||||
}
|
||||
|
||||
func (_c *MockMixCoordClient_ValidateAnalyzer_Call) Run(run func(ctx context.Context, in *querypb.ValidateAnalyzerRequest, opts ...grpc.CallOption)) *MockMixCoordClient_ValidateAnalyzer_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
variadicArgs := make([]grpc.CallOption, len(args)-2)
|
||||
for i, a := range args[2:] {
|
||||
if a != nil {
|
||||
variadicArgs[i] = a.(grpc.CallOption)
|
||||
}
|
||||
}
|
||||
run(args[0].(context.Context), args[1].(*querypb.ValidateAnalyzerRequest), variadicArgs...)
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMixCoordClient_ValidateAnalyzer_Call) Return(_a0 *commonpb.Status, _a1 error) *MockMixCoordClient_ValidateAnalyzer_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockMixCoordClient_ValidateAnalyzer_Call) RunAndReturn(run func(context.Context, *querypb.ValidateAnalyzerRequest, ...grpc.CallOption) (*commonpb.Status, error)) *MockMixCoordClient_ValidateAnalyzer_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// WatchChannels provides a mock function with given fields: ctx, in, opts
|
||||
func (_m *MockMixCoordClient) WatchChannels(ctx context.Context, in *datapb.WatchChannelsRequest, opts ...grpc.CallOption) (*datapb.WatchChannelsResponse, error) {
|
||||
_va := make([]interface{}, len(opts))
|
||||
|
||||
@ -1599,6 +1599,65 @@ func (_c *MockQueryCoord_ResumeNode_Call) RunAndReturn(run func(context.Context,
|
||||
return _c
|
||||
}
|
||||
|
||||
// RunAnalyzer provides a mock function with given fields: _a0, _a1
|
||||
func (_m *MockQueryCoord) RunAnalyzer(_a0 context.Context, _a1 *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) {
|
||||
ret := _m.Called(_a0, _a1)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for RunAnalyzer")
|
||||
}
|
||||
|
||||
var r0 *milvuspb.RunAnalyzerResponse
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error)); ok {
|
||||
return rf(_a0, _a1)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *querypb.RunAnalyzerRequest) *milvuspb.RunAnalyzerResponse); ok {
|
||||
r0 = rf(_a0, _a1)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*milvuspb.RunAnalyzerResponse)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *querypb.RunAnalyzerRequest) error); ok {
|
||||
r1 = rf(_a0, _a1)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockQueryCoord_RunAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RunAnalyzer'
|
||||
type MockQueryCoord_RunAnalyzer_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// RunAnalyzer is a helper method to define mock.On call
|
||||
// - _a0 context.Context
|
||||
// - _a1 *querypb.RunAnalyzerRequest
|
||||
func (_e *MockQueryCoord_Expecter) RunAnalyzer(_a0 interface{}, _a1 interface{}) *MockQueryCoord_RunAnalyzer_Call {
|
||||
return &MockQueryCoord_RunAnalyzer_Call{Call: _e.mock.On("RunAnalyzer", _a0, _a1)}
|
||||
}
|
||||
|
||||
func (_c *MockQueryCoord_RunAnalyzer_Call) Run(run func(_a0 context.Context, _a1 *querypb.RunAnalyzerRequest)) *MockQueryCoord_RunAnalyzer_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(*querypb.RunAnalyzerRequest))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockQueryCoord_RunAnalyzer_Call) Return(_a0 *milvuspb.RunAnalyzerResponse, _a1 error) *MockQueryCoord_RunAnalyzer_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockQueryCoord_RunAnalyzer_Call) RunAndReturn(run func(context.Context, *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error)) *MockQueryCoord_RunAnalyzer_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// SetAddress provides a mock function with given fields: address
|
||||
func (_m *MockQueryCoord) SetAddress(address string) {
|
||||
_m.Called(address)
|
||||
@ -2595,6 +2654,65 @@ func (_c *MockQueryCoord_UpdateStateCode_Call) RunAndReturn(run func(commonpb.St
|
||||
return _c
|
||||
}
|
||||
|
||||
// ValidateAnalyzer provides a mock function with given fields: _a0, _a1
|
||||
func (_m *MockQueryCoord) ValidateAnalyzer(_a0 context.Context, _a1 *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error) {
|
||||
ret := _m.Called(_a0, _a1)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for ValidateAnalyzer")
|
||||
}
|
||||
|
||||
var r0 *commonpb.Status
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error)); ok {
|
||||
return rf(_a0, _a1)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest) *commonpb.Status); ok {
|
||||
r0 = rf(_a0, _a1)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*commonpb.Status)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *querypb.ValidateAnalyzerRequest) error); ok {
|
||||
r1 = rf(_a0, _a1)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockQueryCoord_ValidateAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ValidateAnalyzer'
|
||||
type MockQueryCoord_ValidateAnalyzer_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// ValidateAnalyzer is a helper method to define mock.On call
|
||||
// - _a0 context.Context
|
||||
// - _a1 *querypb.ValidateAnalyzerRequest
|
||||
func (_e *MockQueryCoord_Expecter) ValidateAnalyzer(_a0 interface{}, _a1 interface{}) *MockQueryCoord_ValidateAnalyzer_Call {
|
||||
return &MockQueryCoord_ValidateAnalyzer_Call{Call: _e.mock.On("ValidateAnalyzer", _a0, _a1)}
|
||||
}
|
||||
|
||||
func (_c *MockQueryCoord_ValidateAnalyzer_Call) Run(run func(_a0 context.Context, _a1 *querypb.ValidateAnalyzerRequest)) *MockQueryCoord_ValidateAnalyzer_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(*querypb.ValidateAnalyzerRequest))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockQueryCoord_ValidateAnalyzer_Call) Return(_a0 *commonpb.Status, _a1 error) *MockQueryCoord_ValidateAnalyzer_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockQueryCoord_ValidateAnalyzer_Call) RunAndReturn(run func(context.Context, *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error)) *MockQueryCoord_ValidateAnalyzer_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// NewMockQueryCoord creates a new instance of MockQueryCoord. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
||||
// The first argument is typically a *testing.T value.
|
||||
func NewMockQueryCoord(t interface {
|
||||
|
||||
@ -1926,6 +1926,80 @@ func (_c *MockQueryCoordClient_ResumeNode_Call) RunAndReturn(run func(context.Co
|
||||
return _c
|
||||
}
|
||||
|
||||
// RunAnalyzer provides a mock function with given fields: ctx, in, opts
|
||||
func (_m *MockQueryCoordClient) RunAnalyzer(ctx context.Context, in *querypb.RunAnalyzerRequest, opts ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error) {
|
||||
_va := make([]interface{}, len(opts))
|
||||
for _i := range opts {
|
||||
_va[_i] = opts[_i]
|
||||
}
|
||||
var _ca []interface{}
|
||||
_ca = append(_ca, ctx, in)
|
||||
_ca = append(_ca, _va...)
|
||||
ret := _m.Called(_ca...)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for RunAnalyzer")
|
||||
}
|
||||
|
||||
var r0 *milvuspb.RunAnalyzerResponse
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *querypb.RunAnalyzerRequest, ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error)); ok {
|
||||
return rf(ctx, in, opts...)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *querypb.RunAnalyzerRequest, ...grpc.CallOption) *milvuspb.RunAnalyzerResponse); ok {
|
||||
r0 = rf(ctx, in, opts...)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*milvuspb.RunAnalyzerResponse)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *querypb.RunAnalyzerRequest, ...grpc.CallOption) error); ok {
|
||||
r1 = rf(ctx, in, opts...)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockQueryCoordClient_RunAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RunAnalyzer'
|
||||
type MockQueryCoordClient_RunAnalyzer_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// RunAnalyzer is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - in *querypb.RunAnalyzerRequest
|
||||
// - opts ...grpc.CallOption
|
||||
func (_e *MockQueryCoordClient_Expecter) RunAnalyzer(ctx interface{}, in interface{}, opts ...interface{}) *MockQueryCoordClient_RunAnalyzer_Call {
|
||||
return &MockQueryCoordClient_RunAnalyzer_Call{Call: _e.mock.On("RunAnalyzer",
|
||||
append([]interface{}{ctx, in}, opts...)...)}
|
||||
}
|
||||
|
||||
func (_c *MockQueryCoordClient_RunAnalyzer_Call) Run(run func(ctx context.Context, in *querypb.RunAnalyzerRequest, opts ...grpc.CallOption)) *MockQueryCoordClient_RunAnalyzer_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
variadicArgs := make([]grpc.CallOption, len(args)-2)
|
||||
for i, a := range args[2:] {
|
||||
if a != nil {
|
||||
variadicArgs[i] = a.(grpc.CallOption)
|
||||
}
|
||||
}
|
||||
run(args[0].(context.Context), args[1].(*querypb.RunAnalyzerRequest), variadicArgs...)
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockQueryCoordClient_RunAnalyzer_Call) Return(_a0 *milvuspb.RunAnalyzerResponse, _a1 error) *MockQueryCoordClient_RunAnalyzer_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockQueryCoordClient_RunAnalyzer_Call) RunAndReturn(run func(context.Context, *querypb.RunAnalyzerRequest, ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error)) *MockQueryCoordClient_RunAnalyzer_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// ShowConfigurations provides a mock function with given fields: ctx, in, opts
|
||||
func (_m *MockQueryCoordClient) ShowConfigurations(ctx context.Context, in *internalpb.ShowConfigurationsRequest, opts ...grpc.CallOption) (*internalpb.ShowConfigurationsResponse, error) {
|
||||
_va := make([]interface{}, len(opts))
|
||||
@ -2814,6 +2888,80 @@ func (_c *MockQueryCoordClient_UpdateResourceGroups_Call) RunAndReturn(run func(
|
||||
return _c
|
||||
}
|
||||
|
||||
// ValidateAnalyzer provides a mock function with given fields: ctx, in, opts
|
||||
func (_m *MockQueryCoordClient) ValidateAnalyzer(ctx context.Context, in *querypb.ValidateAnalyzerRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
|
||||
_va := make([]interface{}, len(opts))
|
||||
for _i := range opts {
|
||||
_va[_i] = opts[_i]
|
||||
}
|
||||
var _ca []interface{}
|
||||
_ca = append(_ca, ctx, in)
|
||||
_ca = append(_ca, _va...)
|
||||
ret := _m.Called(_ca...)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for ValidateAnalyzer")
|
||||
}
|
||||
|
||||
var r0 *commonpb.Status
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest, ...grpc.CallOption) (*commonpb.Status, error)); ok {
|
||||
return rf(ctx, in, opts...)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest, ...grpc.CallOption) *commonpb.Status); ok {
|
||||
r0 = rf(ctx, in, opts...)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*commonpb.Status)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *querypb.ValidateAnalyzerRequest, ...grpc.CallOption) error); ok {
|
||||
r1 = rf(ctx, in, opts...)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockQueryCoordClient_ValidateAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ValidateAnalyzer'
|
||||
type MockQueryCoordClient_ValidateAnalyzer_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// ValidateAnalyzer is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - in *querypb.ValidateAnalyzerRequest
|
||||
// - opts ...grpc.CallOption
|
||||
func (_e *MockQueryCoordClient_Expecter) ValidateAnalyzer(ctx interface{}, in interface{}, opts ...interface{}) *MockQueryCoordClient_ValidateAnalyzer_Call {
|
||||
return &MockQueryCoordClient_ValidateAnalyzer_Call{Call: _e.mock.On("ValidateAnalyzer",
|
||||
append([]interface{}{ctx, in}, opts...)...)}
|
||||
}
|
||||
|
||||
func (_c *MockQueryCoordClient_ValidateAnalyzer_Call) Run(run func(ctx context.Context, in *querypb.ValidateAnalyzerRequest, opts ...grpc.CallOption)) *MockQueryCoordClient_ValidateAnalyzer_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
variadicArgs := make([]grpc.CallOption, len(args)-2)
|
||||
for i, a := range args[2:] {
|
||||
if a != nil {
|
||||
variadicArgs[i] = a.(grpc.CallOption)
|
||||
}
|
||||
}
|
||||
run(args[0].(context.Context), args[1].(*querypb.ValidateAnalyzerRequest), variadicArgs...)
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockQueryCoordClient_ValidateAnalyzer_Call) Return(_a0 *commonpb.Status, _a1 error) *MockQueryCoordClient_ValidateAnalyzer_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockQueryCoordClient_ValidateAnalyzer_Call) RunAndReturn(run func(context.Context, *querypb.ValidateAnalyzerRequest, ...grpc.CallOption) (*commonpb.Status, error)) *MockQueryCoordClient_ValidateAnalyzer_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// NewMockQueryCoordClient creates a new instance of MockQueryCoordClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
|
||||
// The first argument is typically a *testing.T value.
|
||||
func NewMockQueryCoordClient(t interface {
|
||||
|
||||
@ -1968,6 +1968,65 @@ func (_c *MockQueryNode_UpdateStateCode_Call) RunAndReturn(run func(commonpb.Sta
|
||||
return _c
|
||||
}
|
||||
|
||||
// ValidateAnalyzer provides a mock function with given fields: _a0, _a1
|
||||
func (_m *MockQueryNode) ValidateAnalyzer(_a0 context.Context, _a1 *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error) {
|
||||
ret := _m.Called(_a0, _a1)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for ValidateAnalyzer")
|
||||
}
|
||||
|
||||
var r0 *commonpb.Status
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error)); ok {
|
||||
return rf(_a0, _a1)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest) *commonpb.Status); ok {
|
||||
r0 = rf(_a0, _a1)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*commonpb.Status)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *querypb.ValidateAnalyzerRequest) error); ok {
|
||||
r1 = rf(_a0, _a1)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockQueryNode_ValidateAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ValidateAnalyzer'
|
||||
type MockQueryNode_ValidateAnalyzer_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// ValidateAnalyzer is a helper method to define mock.On call
|
||||
// - _a0 context.Context
|
||||
// - _a1 *querypb.ValidateAnalyzerRequest
|
||||
func (_e *MockQueryNode_Expecter) ValidateAnalyzer(_a0 interface{}, _a1 interface{}) *MockQueryNode_ValidateAnalyzer_Call {
|
||||
return &MockQueryNode_ValidateAnalyzer_Call{Call: _e.mock.On("ValidateAnalyzer", _a0, _a1)}
|
||||
}
|
||||
|
||||
func (_c *MockQueryNode_ValidateAnalyzer_Call) Run(run func(_a0 context.Context, _a1 *querypb.ValidateAnalyzerRequest)) *MockQueryNode_ValidateAnalyzer_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(*querypb.ValidateAnalyzerRequest))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockQueryNode_ValidateAnalyzer_Call) Return(_a0 *commonpb.Status, _a1 error) *MockQueryNode_ValidateAnalyzer_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockQueryNode_ValidateAnalyzer_Call) RunAndReturn(run func(context.Context, *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error)) *MockQueryNode_ValidateAnalyzer_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// WatchDmChannels provides a mock function with given fields: _a0, _a1
|
||||
func (_m *MockQueryNode) WatchDmChannels(_a0 context.Context, _a1 *querypb.WatchDmChannelsRequest) (*commonpb.Status, error) {
|
||||
ret := _m.Called(_a0, _a1)
|
||||
|
||||
@ -2074,6 +2074,80 @@ func (_c *MockQueryNodeClient_UpdateSchema_Call) RunAndReturn(run func(context.C
|
||||
return _c
|
||||
}
|
||||
|
||||
// ValidateAnalyzer provides a mock function with given fields: ctx, in, opts
|
||||
func (_m *MockQueryNodeClient) ValidateAnalyzer(ctx context.Context, in *querypb.ValidateAnalyzerRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
|
||||
_va := make([]interface{}, len(opts))
|
||||
for _i := range opts {
|
||||
_va[_i] = opts[_i]
|
||||
}
|
||||
var _ca []interface{}
|
||||
_ca = append(_ca, ctx, in)
|
||||
_ca = append(_ca, _va...)
|
||||
ret := _m.Called(_ca...)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for ValidateAnalyzer")
|
||||
}
|
||||
|
||||
var r0 *commonpb.Status
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest, ...grpc.CallOption) (*commonpb.Status, error)); ok {
|
||||
return rf(ctx, in, opts...)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest, ...grpc.CallOption) *commonpb.Status); ok {
|
||||
r0 = rf(ctx, in, opts...)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*commonpb.Status)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *querypb.ValidateAnalyzerRequest, ...grpc.CallOption) error); ok {
|
||||
r1 = rf(ctx, in, opts...)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockQueryNodeClient_ValidateAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ValidateAnalyzer'
|
||||
type MockQueryNodeClient_ValidateAnalyzer_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// ValidateAnalyzer is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - in *querypb.ValidateAnalyzerRequest
|
||||
// - opts ...grpc.CallOption
|
||||
func (_e *MockQueryNodeClient_Expecter) ValidateAnalyzer(ctx interface{}, in interface{}, opts ...interface{}) *MockQueryNodeClient_ValidateAnalyzer_Call {
|
||||
return &MockQueryNodeClient_ValidateAnalyzer_Call{Call: _e.mock.On("ValidateAnalyzer",
|
||||
append([]interface{}{ctx, in}, opts...)...)}
|
||||
}
|
||||
|
||||
func (_c *MockQueryNodeClient_ValidateAnalyzer_Call) Run(run func(ctx context.Context, in *querypb.ValidateAnalyzerRequest, opts ...grpc.CallOption)) *MockQueryNodeClient_ValidateAnalyzer_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
variadicArgs := make([]grpc.CallOption, len(args)-2)
|
||||
for i, a := range args[2:] {
|
||||
if a != nil {
|
||||
variadicArgs[i] = a.(grpc.CallOption)
|
||||
}
|
||||
}
|
||||
run(args[0].(context.Context), args[1].(*querypb.ValidateAnalyzerRequest), variadicArgs...)
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockQueryNodeClient_ValidateAnalyzer_Call) Return(_a0 *commonpb.Status, _a1 error) *MockQueryNodeClient_ValidateAnalyzer_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockQueryNodeClient_ValidateAnalyzer_Call) RunAndReturn(run func(context.Context, *querypb.ValidateAnalyzerRequest, ...grpc.CallOption) (*commonpb.Status, error)) *MockQueryNodeClient_ValidateAnalyzer_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// WatchDmChannels provides a mock function with given fields: ctx, in, opts
|
||||
func (_m *MockQueryNodeClient) WatchDmChannels(ctx context.Context, in *querypb.WatchDmChannelsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
|
||||
_va := make([]interface{}, len(opts))
|
||||
|
||||
@ -46,7 +46,6 @@ import (
|
||||
"github.com/milvus-io/milvus/internal/proxy/privilege"
|
||||
"github.com/milvus-io/milvus/internal/proxy/replicate"
|
||||
"github.com/milvus-io/milvus/internal/types"
|
||||
"github.com/milvus-io/milvus/internal/util/analyzer"
|
||||
"github.com/milvus-io/milvus/internal/util/hookutil"
|
||||
"github.com/milvus-io/milvus/internal/util/segcore"
|
||||
"github.com/milvus-io/milvus/pkg/v2/common"
|
||||
@ -6299,40 +6298,6 @@ func (node *Proxy) OperatePrivilegeGroup(ctx context.Context, req *milvuspb.Oper
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (node *Proxy) runAnalyzer(req *milvuspb.RunAnalyzerRequest) ([]*milvuspb.AnalyzerResult, error) {
|
||||
analyzer, err := analyzer.NewAnalyzer(req.GetAnalyzerParams())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer analyzer.Destroy()
|
||||
|
||||
results := make([]*milvuspb.AnalyzerResult, len(req.GetPlaceholder()))
|
||||
for i, text := range req.GetPlaceholder() {
|
||||
stream := analyzer.NewTokenStream(string(text))
|
||||
defer stream.Destroy()
|
||||
|
||||
results[i] = &milvuspb.AnalyzerResult{
|
||||
Tokens: make([]*milvuspb.AnalyzerToken, 0),
|
||||
}
|
||||
|
||||
for stream.Advance() {
|
||||
var token *milvuspb.AnalyzerToken
|
||||
if req.GetWithDetail() {
|
||||
token = stream.DetailedToken()
|
||||
} else {
|
||||
token = &milvuspb.AnalyzerToken{Token: stream.Token()}
|
||||
}
|
||||
|
||||
if req.GetWithHash() {
|
||||
token.Hash = typeutil.HashString2LessUint32(token.GetToken())
|
||||
}
|
||||
results[i].Tokens = append(results[i].Tokens, token)
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (node *Proxy) RunAnalyzer(ctx context.Context, req *milvuspb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) {
|
||||
ctx, sp := otel.Tracer(typeutil.ProxyRole).Start(ctx, "Proxy-RunAnalyzer")
|
||||
defer sp.End()
|
||||
@ -6350,20 +6315,19 @@ func (node *Proxy) RunAnalyzer(ctx context.Context, req *milvuspb.RunAnalyzerReq
|
||||
}, nil
|
||||
}
|
||||
|
||||
// build and run analyzer at any streaming node/query node
|
||||
// if collection and field not set
|
||||
if req.GetCollectionName() == "" {
|
||||
results, err := node.runAnalyzer(req)
|
||||
if err != nil {
|
||||
return &milvuspb.RunAnalyzerResponse{
|
||||
Status: merr.Status(err),
|
||||
}, nil
|
||||
}
|
||||
|
||||
return &milvuspb.RunAnalyzerResponse{
|
||||
Status: merr.Status(nil),
|
||||
Results: results,
|
||||
}, nil
|
||||
return node.mixCoord.RunAnalyzer(ctx, &querypb.RunAnalyzerRequest{
|
||||
AnalyzerParams: req.GetAnalyzerParams(),
|
||||
Placeholder: req.GetPlaceholder(),
|
||||
WithDetail: req.GetWithDetail(),
|
||||
WithHash: req.GetWithHash(),
|
||||
})
|
||||
}
|
||||
|
||||
// run builded analyzer by delegator
|
||||
// collection must loaded
|
||||
if err := validateRunAnalyzer(req); err != nil {
|
||||
return &milvuspb.RunAnalyzerResponse{
|
||||
Status: merr.Status(merr.WrapErrAsInputError(err)),
|
||||
|
||||
@ -1664,19 +1664,26 @@ func TestRunAnalyzer(t *testing.T) {
|
||||
})
|
||||
|
||||
p.UpdateStateCode(commonpb.StateCode_Healthy)
|
||||
t.Run("run analyzer with default params", func(t *testing.T) {
|
||||
t.Run("run analyzer with mixcoord success", func(t *testing.T) {
|
||||
mockMixcoord := mocks.NewMockMixCoordClient(t)
|
||||
p.mixCoord = mockMixcoord
|
||||
mockMixcoord.EXPECT().RunAnalyzer(mock.Anything, mock.Anything, mock.Anything).Return(&milvuspb.RunAnalyzerResponse{Status: merr.Status(nil)}, nil)
|
||||
|
||||
resp, err := p.RunAnalyzer(context.Background(), &milvuspb.RunAnalyzerRequest{
|
||||
Placeholder: [][]byte{[]byte("test doc")},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, merr.Error(resp.GetStatus()))
|
||||
assert.Equal(t, len(resp.GetResults()[0].GetTokens()), 2)
|
||||
})
|
||||
|
||||
t.Run("run analyzer with invalid params", func(t *testing.T) {
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, merr.Error(resp.GetStatus()))
|
||||
})
|
||||
|
||||
t.Run("run analyzer with mixcoord failed", func(t *testing.T) {
|
||||
mockMixcoord := mocks.NewMockMixCoordClient(t)
|
||||
p.mixCoord = mockMixcoord
|
||||
mockMixcoord.EXPECT().RunAnalyzer(mock.Anything, mock.Anything, mock.Anything).Return(&milvuspb.RunAnalyzerResponse{Status: merr.Status(fmt.Errorf("mock error"))}, nil)
|
||||
|
||||
resp, err := p.RunAnalyzer(context.Background(), &milvuspb.RunAnalyzerRequest{
|
||||
Placeholder: [][]byte{[]byte("test doc")},
|
||||
AnalyzerParams: "invalid json",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Error(t, merr.Error(resp.GetStatus()))
|
||||
|
||||
@ -1642,6 +1642,16 @@ func (coord *MixCoordMock) ListFileResources(ctx context.Context, req *milvuspb.
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (coord *MixCoordMock) RunAnalyzer(ctx context.Context, req *querypb.RunAnalyzerRequest, opts ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error) {
|
||||
return &milvuspb.RunAnalyzerResponse{
|
||||
Status: merr.Success(),
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (coord *MixCoordMock) ValidateAnalyzer(ctx context.Context, req *querypb.ValidateAnalyzerRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
|
||||
return merr.Success(), nil
|
||||
}
|
||||
|
||||
type DescribeCollectionFunc func(ctx context.Context, request *milvuspb.DescribeCollectionRequest, opts ...grpc.CallOption) (*milvuspb.DescribeCollectionResponse, error)
|
||||
|
||||
type ShowPartitionsFunc func(ctx context.Context, request *milvuspb.ShowPartitionsRequest, opts ...grpc.CallOption) (*milvuspb.ShowPartitionsResponse, error)
|
||||
|
||||
@ -325,7 +325,7 @@ func (lb *LBPolicyImpl) Execute(ctx context.Context, workload CollectionWorkLoad
|
||||
return wg.Wait()
|
||||
}
|
||||
|
||||
// Execute will execute any one channel in collection workload
|
||||
// ExecuteOneChannel will execute at any one channel in collection
|
||||
func (lb *LBPolicyImpl) ExecuteOneChannel(ctx context.Context, workload CollectionWorkLoad) error {
|
||||
channelList, err := lb.GetShardLeaderList(ctx, workload.Db, workload.CollectionName, workload.CollectionID, true)
|
||||
if err != nil {
|
||||
|
||||
@ -40,7 +40,6 @@ import (
|
||||
"github.com/milvus-io/milvus/internal/parser/planparserv2"
|
||||
"github.com/milvus-io/milvus/internal/proxy/privilege"
|
||||
"github.com/milvus-io/milvus/internal/types"
|
||||
"github.com/milvus-io/milvus/internal/util/analyzer"
|
||||
"github.com/milvus-io/milvus/internal/util/function/embedding"
|
||||
"github.com/milvus-io/milvus/internal/util/hookutil"
|
||||
"github.com/milvus-io/milvus/internal/util/indexparamcheck"
|
||||
@ -632,10 +631,6 @@ func ValidateField(field *schemapb.FieldSchema, schema *schemapb.CollectionSchem
|
||||
if err = ValidateAutoIndexMmapConfig(isVectorType, indexParams); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := validateAnalyzer(schema, field); err != nil {
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -703,107 +698,6 @@ func ValidateStructArrayField(structArrayField *schemapb.StructArrayFieldSchema,
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateMultiAnalyzerParams(params string, coll *schemapb.CollectionSchema) error {
|
||||
var m map[string]json.RawMessage
|
||||
var analyzerMap map[string]json.RawMessage
|
||||
var mFileName string
|
||||
|
||||
err := json.Unmarshal([]byte(params), &m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
mfield, ok := m["by_field"]
|
||||
if !ok {
|
||||
return fmt.Errorf("multi analyzer params now must set by_field to specify with field decide analyzer")
|
||||
}
|
||||
|
||||
err = json.Unmarshal(mfield, &mFileName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("multi analyzer params by_field must be string but now: %s", mfield)
|
||||
}
|
||||
|
||||
// check field exist
|
||||
fieldExist := false
|
||||
for _, field := range coll.GetFields() {
|
||||
if field.GetName() == mFileName {
|
||||
// only support string field now
|
||||
if field.GetDataType() != schemapb.DataType_VarChar {
|
||||
return fmt.Errorf("multi analyzer params now only support by string field, but field %s is not string", field.GetName())
|
||||
}
|
||||
fieldExist = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !fieldExist {
|
||||
return fmt.Errorf("multi analyzer dependent field %s not exist in collection %s", string(mfield), coll.GetName())
|
||||
}
|
||||
|
||||
if value, ok := m["alias"]; ok {
|
||||
mapping := map[string]string{}
|
||||
err = json.Unmarshal(value, &mapping)
|
||||
if err != nil {
|
||||
return fmt.Errorf("multi analyzer alias must be string map but now: %s", value)
|
||||
}
|
||||
}
|
||||
|
||||
analyzers, ok := m["analyzers"]
|
||||
if !ok {
|
||||
return fmt.Errorf("multi analyzer params must set analyzers ")
|
||||
}
|
||||
|
||||
err = json.Unmarshal(analyzers, &analyzerMap)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unmarshal analyzers failed: %s", err)
|
||||
}
|
||||
|
||||
hasDefault := false
|
||||
for name, params := range analyzerMap {
|
||||
if err := analyzer.ValidateAnalyzer(string(params)); err != nil {
|
||||
return fmt.Errorf("analyzer %s params invalid: %s", name, err)
|
||||
}
|
||||
if name == "default" {
|
||||
hasDefault = true
|
||||
}
|
||||
}
|
||||
|
||||
if !hasDefault {
|
||||
return fmt.Errorf("multi analyzer must set default analyzer for all unknown value")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateAnalyzer(collSchema *schemapb.CollectionSchema, fieldSchema *schemapb.FieldSchema) error {
|
||||
h := typeutil.CreateFieldSchemaHelper(fieldSchema)
|
||||
if !h.EnableMatch() && !wasBm25FunctionInputField(collSchema, fieldSchema) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if !h.EnableAnalyzer() {
|
||||
return fmt.Errorf("field %s is set to enable match or bm25 function but not enable analyzer", fieldSchema.Name)
|
||||
}
|
||||
|
||||
if params, ok := h.GetMultiAnalyzerParams(); ok {
|
||||
if h.EnableMatch() {
|
||||
return fmt.Errorf("multi analyzer now only support for bm25, but now field %s enable match", fieldSchema.Name)
|
||||
}
|
||||
if h.HasAnalyzerParams() {
|
||||
return fmt.Errorf("field %s analyzer params should be none if has multi analyzer params", fieldSchema.Name)
|
||||
}
|
||||
|
||||
return validateMultiAnalyzerParams(params, collSchema)
|
||||
}
|
||||
|
||||
for _, kv := range fieldSchema.GetTypeParams() {
|
||||
if kv.GetKey() == "analyzer_params" {
|
||||
return analyzer.ValidateAnalyzer(kv.Value)
|
||||
}
|
||||
}
|
||||
// return nil when use default analyzer
|
||||
return nil
|
||||
}
|
||||
|
||||
func validatePrimaryKey(coll *schemapb.CollectionSchema) error {
|
||||
idx := -1
|
||||
for i, field := range coll.Fields {
|
||||
@ -1222,7 +1116,7 @@ func fillFieldPropertiesBySchema(columns []*schemapb.FieldData, schema *schemapb
|
||||
expectColumnNum := 0
|
||||
for _, field := range schema.GetFields() {
|
||||
fieldName2Schema[field.Name] = field
|
||||
if !IsBM25FunctionOutputField(field, schema) {
|
||||
if !typeutil.IsBM25FunctionOutputField(field, schema) {
|
||||
expectColumnNum++
|
||||
}
|
||||
}
|
||||
@ -1828,12 +1722,12 @@ func checkFieldsDataBySchema(allFields []*schemapb.FieldSchema, schema *schemapb
|
||||
if fieldSchema.GetDefaultValue() != nil && fieldSchema.IsPrimaryKey {
|
||||
return merr.WrapErrParameterInvalidMsg("primary key can't be with default value")
|
||||
}
|
||||
if (fieldSchema.IsPrimaryKey && fieldSchema.AutoID && !Params.ProxyCfg.SkipAutoIDCheck.GetAsBool() && needAutoGenPk && inInsert) || IsBM25FunctionOutputField(fieldSchema, schema) {
|
||||
if (fieldSchema.IsPrimaryKey && fieldSchema.AutoID && !Params.ProxyCfg.SkipAutoIDCheck.GetAsBool() && needAutoGenPk && inInsert) || typeutil.IsBM25FunctionOutputField(fieldSchema, schema) {
|
||||
// when inInsert, no need to pass when pk is autoid and SkipAutoIDCheck is false
|
||||
autoGenFieldNum++
|
||||
}
|
||||
if _, ok := dataNameSet[fieldSchema.GetName()]; !ok {
|
||||
if (fieldSchema.IsPrimaryKey && fieldSchema.AutoID && !Params.ProxyCfg.SkipAutoIDCheck.GetAsBool() && needAutoGenPk && inInsert) || IsBM25FunctionOutputField(fieldSchema, schema) {
|
||||
if (fieldSchema.IsPrimaryKey && fieldSchema.AutoID && !Params.ProxyCfg.SkipAutoIDCheck.GetAsBool() && needAutoGenPk && inInsert) || typeutil.IsBM25FunctionOutputField(fieldSchema, schema) {
|
||||
// autoGenField
|
||||
continue
|
||||
}
|
||||
@ -2044,7 +1938,7 @@ func LackOfFieldsDataBySchema(schema *schemapb.CollectionSchema, fieldsData []*s
|
||||
|
||||
if _, ok := dataNameMap[fieldSchema.GetName()]; !ok {
|
||||
if (fieldSchema.IsPrimaryKey && fieldSchema.AutoID && !Params.ProxyCfg.SkipAutoIDCheck.GetAsBool() && skipPkFieldCheck) ||
|
||||
IsBM25FunctionOutputField(fieldSchema, schema) ||
|
||||
typeutil.IsBM25FunctionOutputField(fieldSchema, schema) ||
|
||||
(skipDynamicFieldCheck && fieldSchema.GetIsDynamic()) {
|
||||
// autoGenField
|
||||
continue
|
||||
@ -2707,24 +2601,6 @@ func GetReplicateID(ctx context.Context, database, collectionName string) (strin
|
||||
return replicateID, nil
|
||||
}
|
||||
|
||||
func IsBM25FunctionOutputField(field *schemapb.FieldSchema, collSchema *schemapb.CollectionSchema) bool {
|
||||
if !(field.GetIsFunctionOutput() && field.GetDataType() == schemapb.DataType_SparseFloatVector) {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, fSchema := range collSchema.Functions {
|
||||
if fSchema.Type == schemapb.FunctionType_BM25 {
|
||||
if len(fSchema.OutputFieldNames) != 0 && field.Name == fSchema.OutputFieldNames[0] {
|
||||
return true
|
||||
}
|
||||
if len(fSchema.OutputFieldIds) != 0 && field.FieldID == fSchema.OutputFieldIds[0] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func GetFunctionOutputFields(collSchema *schemapb.CollectionSchema) []string {
|
||||
fields := make([]string, 0)
|
||||
for _, fSchema := range collSchema.Functions {
|
||||
|
||||
@ -3086,25 +3086,6 @@ func TestValidateFunctionBasicParams(t *testing.T) {
|
||||
})
|
||||
}
|
||||
|
||||
func TestIsBM25FunctionOutputField(t *testing.T) {
|
||||
schema := &schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{Name: "input_field", DataType: schemapb.DataType_VarChar, TypeParams: []*commonpb.KeyValuePair{{Key: "enable_analyzer", Value: "true"}}},
|
||||
{Name: "output_field", DataType: schemapb.DataType_SparseFloatVector, IsFunctionOutput: true},
|
||||
},
|
||||
Functions: []*schemapb.FunctionSchema{
|
||||
{
|
||||
Name: "bm25_func",
|
||||
Type: schemapb.FunctionType_BM25,
|
||||
InputFieldNames: []string{"input_field"},
|
||||
OutputFieldNames: []string{"output_field"},
|
||||
},
|
||||
},
|
||||
}
|
||||
assert.False(t, IsBM25FunctionOutputField(schema.Fields[0], schema))
|
||||
assert.True(t, IsBM25FunctionOutputField(schema.Fields[1], schema))
|
||||
}
|
||||
|
||||
func TestComputeRecall(t *testing.T) {
|
||||
t.Run("normal case1", func(t *testing.T) {
|
||||
result1 := &schemapb.SearchResultData{
|
||||
|
||||
@ -1211,12 +1211,3 @@ func newValidateUtil(opts ...validateOption) *validateUtil {
|
||||
func ValidateAutoIndexMmapConfig(isVectorField bool, indexParams map[string]string) error {
|
||||
return common.ValidateAutoIndexMmapConfig(paramtable.Get().AutoIndexConfig.Enable.GetAsBool(), isVectorField, indexParams)
|
||||
}
|
||||
|
||||
func wasBm25FunctionInputField(coll *schemapb.CollectionSchema, field *schemapb.FieldSchema) bool {
|
||||
for _, fun := range coll.GetFunctions() {
|
||||
if fun.GetType() == schemapb.FunctionType_BM25 && field.GetName() == fun.GetInputFieldNames()[0] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@ -1598,6 +1598,65 @@ func (_c *MockQueryNodeServer_UpdateSchema_Call) RunAndReturn(run func(context.C
|
||||
return _c
|
||||
}
|
||||
|
||||
// ValidateAnalyzer provides a mock function with given fields: _a0, _a1
|
||||
func (_m *MockQueryNodeServer) ValidateAnalyzer(_a0 context.Context, _a1 *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error) {
|
||||
ret := _m.Called(_a0, _a1)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for ValidateAnalyzer")
|
||||
}
|
||||
|
||||
var r0 *commonpb.Status
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error)); ok {
|
||||
return rf(_a0, _a1)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest) *commonpb.Status); ok {
|
||||
r0 = rf(_a0, _a1)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*commonpb.Status)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *querypb.ValidateAnalyzerRequest) error); ok {
|
||||
r1 = rf(_a0, _a1)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockQueryNodeServer_ValidateAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ValidateAnalyzer'
|
||||
type MockQueryNodeServer_ValidateAnalyzer_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// ValidateAnalyzer is a helper method to define mock.On call
|
||||
// - _a0 context.Context
|
||||
// - _a1 *querypb.ValidateAnalyzerRequest
|
||||
func (_e *MockQueryNodeServer_Expecter) ValidateAnalyzer(_a0 interface{}, _a1 interface{}) *MockQueryNodeServer_ValidateAnalyzer_Call {
|
||||
return &MockQueryNodeServer_ValidateAnalyzer_Call{Call: _e.mock.On("ValidateAnalyzer", _a0, _a1)}
|
||||
}
|
||||
|
||||
func (_c *MockQueryNodeServer_ValidateAnalyzer_Call) Run(run func(_a0 context.Context, _a1 *querypb.ValidateAnalyzerRequest)) *MockQueryNodeServer_ValidateAnalyzer_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(*querypb.ValidateAnalyzerRequest))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockQueryNodeServer_ValidateAnalyzer_Call) Return(_a0 *commonpb.Status, _a1 error) *MockQueryNodeServer_ValidateAnalyzer_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockQueryNodeServer_ValidateAnalyzer_Call) RunAndReturn(run func(context.Context, *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error)) *MockQueryNodeServer_ValidateAnalyzer_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// WatchDmChannels provides a mock function with given fields: _a0, _a1
|
||||
func (_m *MockQueryNodeServer) WatchDmChannels(_a0 context.Context, _a1 *querypb.WatchDmChannelsRequest) (*commonpb.Status, error) {
|
||||
ret := _m.Called(_a0, _a1)
|
||||
|
||||
@ -129,6 +129,10 @@ type Server struct {
|
||||
proxyClientManager proxyutil.ProxyClientManagerInterface
|
||||
|
||||
metricsRequest *metricsinfo.MetricsRequest
|
||||
|
||||
// for balance streaming node request
|
||||
// now only used for run analyzer and validate analyzer
|
||||
nodeIdx atomic.Uint32
|
||||
}
|
||||
|
||||
func NewQueryCoord(ctx context.Context) (*Server, error) {
|
||||
|
||||
@ -29,6 +29,7 @@ import (
|
||||
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
|
||||
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
|
||||
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
|
||||
"github.com/milvus-io/milvus/internal/coordinator/snmanager"
|
||||
"github.com/milvus-io/milvus/internal/querycoordv2/job"
|
||||
"github.com/milvus-io/milvus/internal/querycoordv2/meta"
|
||||
"github.com/milvus-io/milvus/internal/querycoordv2/utils"
|
||||
@ -1289,3 +1290,47 @@ func (s *Server) ListLoadedSegments(ctx context.Context, req *querypb.ListLoaded
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *Server) RunAnalyzer(ctx context.Context, req *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) {
|
||||
if err := merr.CheckHealthy(s.State()); err != nil {
|
||||
return &milvuspb.RunAnalyzerResponse{
|
||||
Status: merr.Status(errors.Wrap(err, "failed to run analyzer")),
|
||||
}, nil
|
||||
}
|
||||
|
||||
nodeIDs := snmanager.StaticStreamingNodeManager.GetStreamingQueryNodeIDs().Collect()
|
||||
|
||||
if len(nodeIDs) == 0 {
|
||||
return &milvuspb.RunAnalyzerResponse{
|
||||
Status: merr.Status(errors.New("failed to validate analyzer, no delegator")),
|
||||
}, nil
|
||||
}
|
||||
|
||||
idx := s.nodeIdx.Inc() % uint32(len(nodeIDs))
|
||||
resp, err := s.cluster.RunAnalyzer(ctx, nodeIDs[idx], req)
|
||||
if err != nil {
|
||||
return &milvuspb.RunAnalyzerResponse{
|
||||
Status: merr.Status(err),
|
||||
}, nil
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
func (s *Server) ValidateAnalyzer(ctx context.Context, req *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error) {
|
||||
if err := merr.CheckHealthy(s.State()); err != nil {
|
||||
return merr.Status(errors.Wrap(err, "failed to validate analyzer")), nil
|
||||
}
|
||||
|
||||
nodeIDs := snmanager.StaticStreamingNodeManager.GetStreamingQueryNodeIDs().Collect()
|
||||
|
||||
if len(nodeIDs) == 0 {
|
||||
return merr.Status(errors.New("failed to validate analyzer, no delegator")), nil
|
||||
}
|
||||
|
||||
idx := s.nodeIdx.Inc() % uint32(len(nodeIDs))
|
||||
resp, err := s.cluster.ValidateAnalyzer(ctx, nodeIDs[idx], req)
|
||||
if err != nil {
|
||||
return merr.Status(err), nil
|
||||
}
|
||||
return resp, nil
|
||||
}
|
||||
|
||||
@ -53,6 +53,8 @@ type Cluster interface {
|
||||
SyncDistribution(ctx context.Context, nodeID int64, req *querypb.SyncDistributionRequest) (*commonpb.Status, error)
|
||||
GetComponentStates(ctx context.Context, nodeID int64) (*milvuspb.ComponentStates, error)
|
||||
DropIndex(ctx context.Context, nodeID int64, req *querypb.DropIndexRequest) (*commonpb.Status, error)
|
||||
RunAnalyzer(ctx context.Context, nodeID int64, req *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error)
|
||||
ValidateAnalyzer(ctx context.Context, nodeID int64, req *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error)
|
||||
Start()
|
||||
Stop()
|
||||
}
|
||||
@ -283,6 +285,36 @@ func (c *QueryCluster) DropIndex(ctx context.Context, nodeID int64, req *querypb
|
||||
return resp, err
|
||||
}
|
||||
|
||||
func (c *QueryCluster) RunAnalyzer(ctx context.Context, nodeID int64, req *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) {
|
||||
var (
|
||||
resp *milvuspb.RunAnalyzerResponse
|
||||
err error
|
||||
)
|
||||
|
||||
sendErr := c.send(ctx, nodeID, func(cli types.QueryNodeClient) {
|
||||
resp, err = cli.RunAnalyzer(ctx, req)
|
||||
})
|
||||
if sendErr != nil {
|
||||
return nil, sendErr
|
||||
}
|
||||
return resp, err
|
||||
}
|
||||
|
||||
func (c *QueryCluster) ValidateAnalyzer(ctx context.Context, nodeID int64, req *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error) {
|
||||
var (
|
||||
resp *commonpb.Status
|
||||
err error
|
||||
)
|
||||
|
||||
sendErr := c.send(ctx, nodeID, func(cli types.QueryNodeClient) {
|
||||
resp, err = cli.ValidateAnalyzer(ctx, req)
|
||||
})
|
||||
if sendErr != nil {
|
||||
return nil, sendErr
|
||||
}
|
||||
return resp, err
|
||||
}
|
||||
|
||||
func (c *QueryCluster) send(ctx context.Context, nodeID int64, fn func(cli types.QueryNodeClient)) error {
|
||||
node := c.nodeManager.Get(nodeID)
|
||||
if node == nil {
|
||||
|
||||
@ -506,6 +506,66 @@ func (_c *MockCluster_ReleaseSegments_Call) RunAndReturn(run func(context.Contex
|
||||
return _c
|
||||
}
|
||||
|
||||
// RunAnalyzer provides a mock function with given fields: ctx, nodeID, req
|
||||
func (_m *MockCluster) RunAnalyzer(ctx context.Context, nodeID int64, req *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) {
|
||||
ret := _m.Called(ctx, nodeID, req)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for RunAnalyzer")
|
||||
}
|
||||
|
||||
var r0 *milvuspb.RunAnalyzerResponse
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error)); ok {
|
||||
return rf(ctx, nodeID, req)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, *querypb.RunAnalyzerRequest) *milvuspb.RunAnalyzerResponse); ok {
|
||||
r0 = rf(ctx, nodeID, req)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*milvuspb.RunAnalyzerResponse)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, int64, *querypb.RunAnalyzerRequest) error); ok {
|
||||
r1 = rf(ctx, nodeID, req)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockCluster_RunAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RunAnalyzer'
|
||||
type MockCluster_RunAnalyzer_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// RunAnalyzer is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - nodeID int64
|
||||
// - req *querypb.RunAnalyzerRequest
|
||||
func (_e *MockCluster_Expecter) RunAnalyzer(ctx interface{}, nodeID interface{}, req interface{}) *MockCluster_RunAnalyzer_Call {
|
||||
return &MockCluster_RunAnalyzer_Call{Call: _e.mock.On("RunAnalyzer", ctx, nodeID, req)}
|
||||
}
|
||||
|
||||
func (_c *MockCluster_RunAnalyzer_Call) Run(run func(ctx context.Context, nodeID int64, req *querypb.RunAnalyzerRequest)) *MockCluster_RunAnalyzer_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(int64), args[2].(*querypb.RunAnalyzerRequest))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockCluster_RunAnalyzer_Call) Return(_a0 *milvuspb.RunAnalyzerResponse, _a1 error) *MockCluster_RunAnalyzer_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockCluster_RunAnalyzer_Call) RunAndReturn(run func(context.Context, int64, *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error)) *MockCluster_RunAnalyzer_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// Start provides a mock function with no fields
|
||||
func (_m *MockCluster) Start() {
|
||||
_m.Called()
|
||||
@ -690,6 +750,66 @@ func (_c *MockCluster_UnsubDmChannel_Call) RunAndReturn(run func(context.Context
|
||||
return _c
|
||||
}
|
||||
|
||||
// ValidateAnalyzer provides a mock function with given fields: ctx, nodeID, req
|
||||
func (_m *MockCluster) ValidateAnalyzer(ctx context.Context, nodeID int64, req *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error) {
|
||||
ret := _m.Called(ctx, nodeID, req)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for ValidateAnalyzer")
|
||||
}
|
||||
|
||||
var r0 *commonpb.Status
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error)); ok {
|
||||
return rf(ctx, nodeID, req)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, int64, *querypb.ValidateAnalyzerRequest) *commonpb.Status); ok {
|
||||
r0 = rf(ctx, nodeID, req)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*commonpb.Status)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, int64, *querypb.ValidateAnalyzerRequest) error); ok {
|
||||
r1 = rf(ctx, nodeID, req)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockCluster_ValidateAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ValidateAnalyzer'
|
||||
type MockCluster_ValidateAnalyzer_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// ValidateAnalyzer is a helper method to define mock.On call
|
||||
// - ctx context.Context
|
||||
// - nodeID int64
|
||||
// - req *querypb.ValidateAnalyzerRequest
|
||||
func (_e *MockCluster_Expecter) ValidateAnalyzer(ctx interface{}, nodeID interface{}, req interface{}) *MockCluster_ValidateAnalyzer_Call {
|
||||
return &MockCluster_ValidateAnalyzer_Call{Call: _e.mock.On("ValidateAnalyzer", ctx, nodeID, req)}
|
||||
}
|
||||
|
||||
func (_c *MockCluster_ValidateAnalyzer_Call) Run(run func(ctx context.Context, nodeID int64, req *querypb.ValidateAnalyzerRequest)) *MockCluster_ValidateAnalyzer_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(int64), args[2].(*querypb.ValidateAnalyzerRequest))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockCluster_ValidateAnalyzer_Call) Return(_a0 *commonpb.Status, _a1 error) *MockCluster_ValidateAnalyzer_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockCluster_ValidateAnalyzer_Call) RunAndReturn(run func(context.Context, int64, *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error)) *MockCluster_ValidateAnalyzer_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// WatchDmChannels provides a mock function with given fields: ctx, nodeID, req
|
||||
func (_m *MockCluster) WatchDmChannels(ctx context.Context, nodeID int64, req *querypb.WatchDmChannelsRequest) (*commonpb.Status, error) {
|
||||
ret := _m.Called(ctx, nodeID, req)
|
||||
|
||||
@ -37,6 +37,7 @@ import (
|
||||
"github.com/milvus-io/milvus/internal/querynodev2/segments"
|
||||
"github.com/milvus-io/milvus/internal/querynodev2/tasks"
|
||||
"github.com/milvus-io/milvus/internal/storage"
|
||||
"github.com/milvus-io/milvus/internal/util/analyzer"
|
||||
"github.com/milvus-io/milvus/internal/util/searchutil/scheduler"
|
||||
"github.com/milvus-io/milvus/internal/util/streamrpc"
|
||||
"github.com/milvus-io/milvus/pkg/v2/common"
|
||||
@ -1572,7 +1573,60 @@ func (node *QueryNode) DeleteBatch(ctx context.Context, req *querypb.DeleteBatch
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (node *QueryNode) runAnalyzer(req *querypb.RunAnalyzerRequest) ([]*milvuspb.AnalyzerResult, error) {
|
||||
tokenizer, err := analyzer.NewAnalyzer(req.GetAnalyzerParams())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer tokenizer.Destroy()
|
||||
|
||||
results := make([]*milvuspb.AnalyzerResult, len(req.GetPlaceholder()))
|
||||
for i, text := range req.GetPlaceholder() {
|
||||
stream := tokenizer.NewTokenStream(string(text))
|
||||
defer stream.Destroy()
|
||||
|
||||
results[i] = &milvuspb.AnalyzerResult{
|
||||
Tokens: make([]*milvuspb.AnalyzerToken, 0),
|
||||
}
|
||||
|
||||
for stream.Advance() {
|
||||
var token *milvuspb.AnalyzerToken
|
||||
if req.GetWithDetail() {
|
||||
token = stream.DetailedToken()
|
||||
} else {
|
||||
token = &milvuspb.AnalyzerToken{Token: stream.Token()}
|
||||
}
|
||||
|
||||
if req.GetWithHash() {
|
||||
token.Hash = typeutil.HashString2LessUint32(token.GetToken())
|
||||
}
|
||||
results[i].Tokens = append(results[i].Tokens, token)
|
||||
}
|
||||
}
|
||||
return results, nil
|
||||
}
|
||||
|
||||
func (node *QueryNode) RunAnalyzer(ctx context.Context, req *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) {
|
||||
// check node healthy
|
||||
if err := node.lifetime.Add(merr.IsHealthy); err != nil {
|
||||
return &milvuspb.RunAnalyzerResponse{
|
||||
Status: merr.Status(err),
|
||||
}, nil
|
||||
}
|
||||
defer node.lifetime.Done()
|
||||
|
||||
// build and run analyzer by analyzer params
|
||||
// if channel not set
|
||||
if req.GetChannel() == "" {
|
||||
result, err := node.runAnalyzer(req)
|
||||
return &milvuspb.RunAnalyzerResponse{
|
||||
Status: merr.Status(err),
|
||||
Results: result,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// run analyzer by delegator
|
||||
// get delegator
|
||||
sd, ok := node.delegators.Get(req.GetChannel())
|
||||
if !ok {
|
||||
@ -1597,6 +1651,26 @@ func (node *QueryNode) RunAnalyzer(ctx context.Context, req *querypb.RunAnalyzer
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (node *QueryNode) ValidateAnalyzer(ctx context.Context, req *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error) {
|
||||
// check node healthy
|
||||
if err := node.lifetime.Add(merr.IsHealthy); err != nil {
|
||||
return merr.Status(err), nil
|
||||
}
|
||||
defer node.lifetime.Done()
|
||||
|
||||
for _, info := range req.AnalyzerInfos {
|
||||
err := analyzer.ValidateAnalyzer(info.GetParams())
|
||||
if err != nil {
|
||||
if info.GetName() != "" {
|
||||
return merr.Status(merr.WrapErrParameterInvalidMsg("validate analyzer failed for field: %s, name: %s, error: %v", info.GetField(), info.GetName(), err)), nil
|
||||
}
|
||||
return merr.Status(merr.WrapErrParameterInvalidMsg("validate analyzer failed for field: %s, error: %v", info.GetField(), err)), nil
|
||||
}
|
||||
}
|
||||
|
||||
return merr.Status(nil), nil
|
||||
}
|
||||
|
||||
type deleteRequestStringer struct {
|
||||
*querypb.DeleteRequest
|
||||
}
|
||||
|
||||
@ -2415,6 +2415,60 @@ func (suite *ServiceSuite) TestRunAnalyzer() {
|
||||
})
|
||||
}
|
||||
|
||||
func (suite *ServiceSuite) TestValidateAnalyzer() {
|
||||
ctx := context.Background()
|
||||
|
||||
suite.Run("normal validate", func() {
|
||||
req := &querypb.ValidateAnalyzerRequest{
|
||||
AnalyzerInfos: []*querypb.AnalyzerInfo{
|
||||
{
|
||||
Field: "test_field",
|
||||
Name: "test_analyzer",
|
||||
Params: `{}`,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
resp, err := suite.node.ValidateAnalyzer(ctx, req)
|
||||
suite.Require().NoError(err)
|
||||
suite.Require().Equal(commonpb.ErrorCode_Success, resp.GetErrorCode())
|
||||
})
|
||||
|
||||
suite.Run("invalid analyzer params", func() {
|
||||
req := &querypb.ValidateAnalyzerRequest{
|
||||
AnalyzerInfos: []*querypb.AnalyzerInfo{
|
||||
{
|
||||
Field: "test_field",
|
||||
Name: "test_analyzer",
|
||||
Params: `{"invalid": "params"}`,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
resp, err := suite.node.ValidateAnalyzer(ctx, req)
|
||||
suite.Require().NoError(err)
|
||||
suite.Require().NotEqual(commonpb.ErrorCode_Success, resp.GetErrorCode())
|
||||
})
|
||||
|
||||
suite.Run("abnormal node", func() {
|
||||
suite.node.UpdateStateCode(commonpb.StateCode_Abnormal)
|
||||
defer suite.node.UpdateStateCode(commonpb.StateCode_Healthy)
|
||||
|
||||
req := &querypb.ValidateAnalyzerRequest{
|
||||
AnalyzerInfos: []*querypb.AnalyzerInfo{
|
||||
{
|
||||
Field: "test_field",
|
||||
Params: `{}`,
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
resp, err := suite.node.ValidateAnalyzer(ctx, req)
|
||||
suite.Require().NoError(err)
|
||||
suite.Require().NotEqual(commonpb.ErrorCode_Success, resp.GetErrorCode())
|
||||
})
|
||||
}
|
||||
|
||||
func TestQueryNodeService(t *testing.T) {
|
||||
wal := mock_streaming.NewMockWALAccesser(t)
|
||||
local := mock_streaming.NewMockLocal(t)
|
||||
|
||||
@ -33,6 +33,7 @@ import (
|
||||
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
|
||||
"github.com/milvus-io/milvus/internal/coordinator/snmanager"
|
||||
"github.com/milvus-io/milvus/internal/distributed/streaming"
|
||||
"github.com/milvus-io/milvus/internal/json"
|
||||
"github.com/milvus-io/milvus/internal/metastore/model"
|
||||
"github.com/milvus-io/milvus/internal/util/hookutil"
|
||||
"github.com/milvus-io/milvus/internal/util/proxyutil"
|
||||
@ -41,6 +42,7 @@ import (
|
||||
"github.com/milvus-io/milvus/pkg/v2/log"
|
||||
ms "github.com/milvus-io/milvus/pkg/v2/mq/msgstream"
|
||||
pb "github.com/milvus-io/milvus/pkg/v2/proto/etcdpb"
|
||||
"github.com/milvus-io/milvus/pkg/v2/proto/querypb"
|
||||
"github.com/milvus-io/milvus/pkg/v2/streaming/util/message"
|
||||
"github.com/milvus-io/milvus/pkg/v2/streaming/util/message/adaptor"
|
||||
"github.com/milvus-io/milvus/pkg/v2/util/commonpbutil"
|
||||
@ -116,6 +118,7 @@ func (t *createCollectionTask) validate(ctx context.Context) error {
|
||||
if t.Req.GetNumPartitions() > 0 {
|
||||
newPartNum = t.Req.GetNumPartitions()
|
||||
}
|
||||
|
||||
return checkGeneralCapacity(t.ctx, 1, newPartNum, t.Req.GetShardsNum(), t.core)
|
||||
}
|
||||
|
||||
@ -210,6 +213,29 @@ func (t *createCollectionTask) validateSchema(ctx context.Context, schema *schem
|
||||
return err
|
||||
}
|
||||
|
||||
// check analyzer was vaild
|
||||
analyzerInfos := make([]*querypb.AnalyzerInfo, 0)
|
||||
for _, field := range schema.GetFields() {
|
||||
err := validateAnalyzer(schema, field, &analyzerInfos)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// validate analyzer params at any streaming node
|
||||
if len(analyzerInfos) > 0 {
|
||||
resp, err := t.core.mixCoord.ValidateAnalyzer(t.ctx, &querypb.ValidateAnalyzerRequest{
|
||||
AnalyzerInfos: analyzerInfos,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err := merr.Error(resp); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
return validateFieldDataType(schema.GetFields())
|
||||
}
|
||||
|
||||
@ -768,3 +794,109 @@ func executeCreateCollectionTaskSteps(ctx context.Context,
|
||||
}, &nullStep{}) // We'll remove the whole collection anyway.
|
||||
return undoTask.Execute(ctx)
|
||||
}
|
||||
|
||||
func validateMultiAnalyzerParams(params string, coll *schemapb.CollectionSchema, fieldSchema *schemapb.FieldSchema, infos *[]*querypb.AnalyzerInfo) error {
|
||||
var m map[string]json.RawMessage
|
||||
var analyzerMap map[string]json.RawMessage
|
||||
var mFileName string
|
||||
|
||||
err := json.Unmarshal([]byte(params), &m)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
mfield, ok := m["by_field"]
|
||||
if !ok {
|
||||
return fmt.Errorf("multi analyzer params now must set by_field to specify with field decide analyzer")
|
||||
}
|
||||
|
||||
err = json.Unmarshal(mfield, &mFileName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("multi analyzer params by_field must be string but now: %s", mfield)
|
||||
}
|
||||
|
||||
// check field exist
|
||||
fieldExist := false
|
||||
for _, field := range coll.GetFields() {
|
||||
if field.GetName() == mFileName {
|
||||
// only support string field now
|
||||
if field.GetDataType() != schemapb.DataType_VarChar {
|
||||
return fmt.Errorf("multi analyzer params now only support by string field, but field %s is not string", field.GetName())
|
||||
}
|
||||
fieldExist = true
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !fieldExist {
|
||||
return fmt.Errorf("multi analyzer dependent field %s not exist in collection %s", string(mfield), coll.GetName())
|
||||
}
|
||||
|
||||
if value, ok := m["alias"]; ok {
|
||||
mapping := map[string]string{}
|
||||
err = json.Unmarshal(value, &mapping)
|
||||
if err != nil {
|
||||
return fmt.Errorf("multi analyzer alias must be string map but now: %s", value)
|
||||
}
|
||||
}
|
||||
|
||||
analyzers, ok := m["analyzers"]
|
||||
if !ok {
|
||||
return fmt.Errorf("multi analyzer params must set analyzers ")
|
||||
}
|
||||
|
||||
err = json.Unmarshal(analyzers, &analyzerMap)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unmarshal analyzers failed: %s", err)
|
||||
}
|
||||
|
||||
hasDefault := false
|
||||
for name, bytes := range analyzerMap {
|
||||
*infos = append(*infos, &querypb.AnalyzerInfo{
|
||||
Name: name,
|
||||
Field: fieldSchema.GetName(),
|
||||
Params: string(bytes),
|
||||
})
|
||||
if name == "default" {
|
||||
hasDefault = true
|
||||
}
|
||||
}
|
||||
|
||||
if !hasDefault {
|
||||
return fmt.Errorf("multi analyzer must set default analyzer for all unknown value")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func validateAnalyzer(collSchema *schemapb.CollectionSchema, fieldSchema *schemapb.FieldSchema, analyzerInfos *[]*querypb.AnalyzerInfo) error {
|
||||
h := typeutil.CreateFieldSchemaHelper(fieldSchema)
|
||||
if !h.EnableMatch() && !typeutil.IsBm25FunctionInputField(collSchema, fieldSchema) {
|
||||
return nil
|
||||
}
|
||||
|
||||
if !h.EnableAnalyzer() {
|
||||
return fmt.Errorf("field %s which has enable_match or is input of BM25 function must also enable_analyzer", fieldSchema.Name)
|
||||
}
|
||||
|
||||
if params, ok := h.GetMultiAnalyzerParams(); ok {
|
||||
if h.EnableMatch() {
|
||||
return fmt.Errorf("multi analyzer now only support for bm25, but now field %s enable match", fieldSchema.Name)
|
||||
}
|
||||
if h.HasAnalyzerParams() {
|
||||
return fmt.Errorf("field %s analyzer params should be none if has multi analyzer params", fieldSchema.Name)
|
||||
}
|
||||
|
||||
return validateMultiAnalyzerParams(params, collSchema, fieldSchema, analyzerInfos)
|
||||
}
|
||||
|
||||
for _, kv := range fieldSchema.GetTypeParams() {
|
||||
if kv.GetKey() == "analyzer_params" {
|
||||
*analyzerInfos = append(*analyzerInfos, &querypb.AnalyzerInfo{
|
||||
Field: fieldSchema.GetName(),
|
||||
Params: kv.GetValue(),
|
||||
})
|
||||
}
|
||||
}
|
||||
// return nil when use default analyzer
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -27,6 +27,7 @@ import (
|
||||
"github.com/cockroachdb/errors"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
"google.golang.org/protobuf/proto"
|
||||
|
||||
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
|
||||
@ -37,6 +38,7 @@ import (
|
||||
mockrootcoord "github.com/milvus-io/milvus/internal/rootcoord/mocks"
|
||||
"github.com/milvus-io/milvus/pkg/v2/common"
|
||||
"github.com/milvus-io/milvus/pkg/v2/proto/datapb"
|
||||
"github.com/milvus-io/milvus/pkg/v2/proto/querypb"
|
||||
"github.com/milvus-io/milvus/pkg/v2/util"
|
||||
"github.com/milvus-io/milvus/pkg/v2/util/funcutil"
|
||||
"github.com/milvus-io/milvus/pkg/v2/util/merr"
|
||||
@ -1800,3 +1802,355 @@ func TestNamespaceProperty(t *testing.T) {
|
||||
assert.Error(t, err)
|
||||
})
|
||||
}
|
||||
|
||||
func Test_validateMultiAnalyzerParams(t *testing.T) {
|
||||
createTestCollectionSchema := func(fields []*schemapb.FieldSchema) *schemapb.CollectionSchema {
|
||||
return &schemapb.CollectionSchema{
|
||||
Name: "test_collection",
|
||||
Fields: fields,
|
||||
}
|
||||
}
|
||||
|
||||
createTestFieldSchema := func(name string, dataType schemapb.DataType) *schemapb.FieldSchema {
|
||||
return &schemapb.FieldSchema{
|
||||
Name: name,
|
||||
DataType: dataType,
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("invalid json params", func(t *testing.T) {
|
||||
coll := createTestCollectionSchema([]*schemapb.FieldSchema{})
|
||||
fieldSchema := createTestFieldSchema("test_field", schemapb.DataType_VarChar)
|
||||
infos := make([]*querypb.AnalyzerInfo, 0)
|
||||
|
||||
err := validateMultiAnalyzerParams("invalid json", coll, fieldSchema, &infos)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("missing by_field", func(t *testing.T) {
|
||||
coll := createTestCollectionSchema([]*schemapb.FieldSchema{})
|
||||
fieldSchema := createTestFieldSchema("test_field", schemapb.DataType_VarChar)
|
||||
infos := make([]*querypb.AnalyzerInfo, 0)
|
||||
|
||||
params := `{"analyzers": {"default": {}}}`
|
||||
err := validateMultiAnalyzerParams(params, coll, fieldSchema, &infos)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("by_field not string", func(t *testing.T) {
|
||||
coll := createTestCollectionSchema([]*schemapb.FieldSchema{})
|
||||
fieldSchema := createTestFieldSchema("test_field", schemapb.DataType_VarChar)
|
||||
infos := make([]*querypb.AnalyzerInfo, 0)
|
||||
|
||||
params := `{"by_field": 123, "analyzers": {"default": {}}}`
|
||||
err := validateMultiAnalyzerParams(params, coll, fieldSchema, &infos)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("by_field references non-existent field", func(t *testing.T) {
|
||||
coll := createTestCollectionSchema([]*schemapb.FieldSchema{
|
||||
createTestFieldSchema("existing_field", schemapb.DataType_VarChar),
|
||||
})
|
||||
fieldSchema := createTestFieldSchema("test_field", schemapb.DataType_VarChar)
|
||||
infos := make([]*querypb.AnalyzerInfo, 0)
|
||||
|
||||
params := `{"by_field": "non_existent_field", "analyzers": {"default": {}}}`
|
||||
err := validateMultiAnalyzerParams(params, coll, fieldSchema, &infos)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("by_field references non-string field", func(t *testing.T) {
|
||||
coll := createTestCollectionSchema([]*schemapb.FieldSchema{
|
||||
createTestFieldSchema("int_field", schemapb.DataType_Int64),
|
||||
})
|
||||
fieldSchema := createTestFieldSchema("test_field", schemapb.DataType_VarChar)
|
||||
infos := make([]*querypb.AnalyzerInfo, 0)
|
||||
|
||||
params := `{"by_field": "int_field", "analyzers": {"default": {}}}`
|
||||
err := validateMultiAnalyzerParams(params, coll, fieldSchema, &infos)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("invalid alias format", func(t *testing.T) {
|
||||
coll := createTestCollectionSchema([]*schemapb.FieldSchema{
|
||||
createTestFieldSchema("string_field", schemapb.DataType_VarChar),
|
||||
})
|
||||
fieldSchema := createTestFieldSchema("test_field", schemapb.DataType_VarChar)
|
||||
infos := make([]*querypb.AnalyzerInfo, 0)
|
||||
|
||||
params := `{"by_field": "string_field", "alias": "invalid_alias", "analyzers": {"default": {}}}`
|
||||
err := validateMultiAnalyzerParams(params, coll, fieldSchema, &infos)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("missing analyzers", func(t *testing.T) {
|
||||
coll := createTestCollectionSchema([]*schemapb.FieldSchema{
|
||||
createTestFieldSchema("string_field", schemapb.DataType_VarChar),
|
||||
})
|
||||
fieldSchema := createTestFieldSchema("test_field", schemapb.DataType_VarChar)
|
||||
infos := make([]*querypb.AnalyzerInfo, 0)
|
||||
|
||||
params := `{"by_field": "string_field"}`
|
||||
err := validateMultiAnalyzerParams(params, coll, fieldSchema, &infos)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("invalid analyzers format", func(t *testing.T) {
|
||||
coll := createTestCollectionSchema([]*schemapb.FieldSchema{
|
||||
createTestFieldSchema("string_field", schemapb.DataType_VarChar),
|
||||
})
|
||||
fieldSchema := createTestFieldSchema("test_field", schemapb.DataType_VarChar)
|
||||
infos := make([]*querypb.AnalyzerInfo, 0)
|
||||
|
||||
params := `{"by_field": "string_field", "analyzers": "invalid_analyzers"}`
|
||||
err := validateMultiAnalyzerParams(params, coll, fieldSchema, &infos)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("missing default analyzer", func(t *testing.T) {
|
||||
coll := createTestCollectionSchema([]*schemapb.FieldSchema{
|
||||
createTestFieldSchema("string_field", schemapb.DataType_VarChar),
|
||||
})
|
||||
fieldSchema := createTestFieldSchema("test_field", schemapb.DataType_VarChar)
|
||||
infos := make([]*querypb.AnalyzerInfo, 0)
|
||||
|
||||
params := `{"by_field": "string_field", "analyzers": {"custom": {}}}`
|
||||
err := validateMultiAnalyzerParams(params, coll, fieldSchema, &infos)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("valid params", func(t *testing.T) {
|
||||
coll := createTestCollectionSchema([]*schemapb.FieldSchema{
|
||||
createTestFieldSchema("string_field", schemapb.DataType_VarChar),
|
||||
})
|
||||
fieldSchema := createTestFieldSchema("test_field", schemapb.DataType_VarChar)
|
||||
infos := make([]*querypb.AnalyzerInfo, 0)
|
||||
|
||||
params := `{
|
||||
"by_field": "string_field",
|
||||
"alias": {"en": "english", "zh": "chinese"},
|
||||
"analyzers": {
|
||||
"default": {"type": "standard"},
|
||||
"english": {"type": "english"},
|
||||
"chinese": {"type": "chinese"}
|
||||
}
|
||||
}`
|
||||
err := validateMultiAnalyzerParams(params, coll, fieldSchema, &infos)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, infos, 3)
|
||||
|
||||
analyzerNames := make(map[string]bool)
|
||||
for _, info := range infos {
|
||||
assert.Equal(t, "test_field", info.Field)
|
||||
analyzerNames[info.Name] = true
|
||||
}
|
||||
assert.True(t, analyzerNames["default"])
|
||||
assert.True(t, analyzerNames["english"])
|
||||
assert.True(t, analyzerNames["chinese"])
|
||||
})
|
||||
|
||||
t.Run("valid params with minimal config", func(t *testing.T) {
|
||||
coll := createTestCollectionSchema([]*schemapb.FieldSchema{
|
||||
createTestFieldSchema("string_field", schemapb.DataType_VarChar),
|
||||
})
|
||||
fieldSchema := createTestFieldSchema("test_field", schemapb.DataType_VarChar)
|
||||
infos := make([]*querypb.AnalyzerInfo, 0)
|
||||
|
||||
params := `{
|
||||
"by_field": "string_field",
|
||||
"analyzers": {"default": {"type": "standard"}}
|
||||
}`
|
||||
err := validateMultiAnalyzerParams(params, coll, fieldSchema, &infos)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, infos, 1)
|
||||
assert.Equal(t, "default", infos[0].Name)
|
||||
assert.Equal(t, "test_field", infos[0].Field)
|
||||
assert.Equal(t, `{"type": "standard"}`, infos[0].Params)
|
||||
})
|
||||
}
|
||||
|
||||
func Test_validateAnalyzer(t *testing.T) {
|
||||
createTestCollectionSchemaWithBM25 := func(fields []*schemapb.FieldSchema, inputFieldName string) *schemapb.CollectionSchema {
|
||||
return &schemapb.CollectionSchema{
|
||||
Name: "test_collection",
|
||||
Fields: fields,
|
||||
Functions: []*schemapb.FunctionSchema{
|
||||
{
|
||||
Name: "bm25_func",
|
||||
Type: schemapb.FunctionType_BM25,
|
||||
InputFieldNames: []string{inputFieldName},
|
||||
OutputFieldNames: []string{"bm25_output"},
|
||||
},
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
createTestFieldSchema := func(name string, dataType schemapb.DataType, typeParams []*commonpb.KeyValuePair) *schemapb.FieldSchema {
|
||||
return &schemapb.FieldSchema{
|
||||
Name: name,
|
||||
DataType: dataType,
|
||||
TypeParams: typeParams,
|
||||
}
|
||||
}
|
||||
|
||||
t.Run("field without enable_match and not BM25 input", func(t *testing.T) {
|
||||
fieldSchema := createTestFieldSchema("text_field", schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
|
||||
{Key: common.MaxLengthKey, Value: "100"},
|
||||
})
|
||||
collSchema := createTestCollectionSchemaWithBM25([]*schemapb.FieldSchema{fieldSchema}, "invalid_field")
|
||||
infos := make([]*querypb.AnalyzerInfo, 0)
|
||||
|
||||
err := validateAnalyzer(collSchema, fieldSchema, &infos)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, infos, 0)
|
||||
})
|
||||
|
||||
t.Run("field with enable_match but no enable_analyzer", func(t *testing.T) {
|
||||
fieldSchema := createTestFieldSchema("text_field", schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
|
||||
{Key: common.MaxLengthKey, Value: "100"},
|
||||
{Key: "enable_match", Value: "true"},
|
||||
})
|
||||
collSchema := createTestCollectionSchemaWithBM25([]*schemapb.FieldSchema{fieldSchema}, "text_field")
|
||||
infos := make([]*querypb.AnalyzerInfo, 0)
|
||||
|
||||
err := validateAnalyzer(collSchema, fieldSchema, &infos)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("field with enable_match and enable_analyzer", func(t *testing.T) {
|
||||
fieldSchema := createTestFieldSchema("text_field", schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
|
||||
{Key: common.MaxLengthKey, Value: "100"},
|
||||
{Key: "enable_match", Value: "true"},
|
||||
{Key: "enable_analyzer", Value: "true"},
|
||||
{Key: "analyzer_params", Value: "{\"type\": \"standard\"}"},
|
||||
})
|
||||
collSchema := createTestCollectionSchemaWithBM25([]*schemapb.FieldSchema{fieldSchema}, "text_field")
|
||||
infos := make([]*querypb.AnalyzerInfo, 0)
|
||||
|
||||
err := validateAnalyzer(collSchema, fieldSchema, &infos)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, infos, 1)
|
||||
assert.Equal(t, "text_field", infos[0].Field)
|
||||
assert.Equal(t, "{\"type\": \"standard\"}", infos[0].Params)
|
||||
})
|
||||
|
||||
t.Run("field with multi analyzer and enable_match", func(t *testing.T) {
|
||||
fieldSchema := createTestFieldSchema("text_field", schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
|
||||
{Key: common.MaxLengthKey, Value: "100"},
|
||||
{Key: "enable_match", Value: "true"},
|
||||
{Key: "enable_analyzer", Value: "true"},
|
||||
{Key: "multi_analyzer_params", Value: `{"by_field": "lang", "analyzers": {"default": "{}"}}`},
|
||||
})
|
||||
collSchema := createTestCollectionSchemaWithBM25([]*schemapb.FieldSchema{
|
||||
fieldSchema,
|
||||
createTestFieldSchema("lang", schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
|
||||
{Key: common.MaxLengthKey, Value: "10"},
|
||||
}),
|
||||
}, "text_field")
|
||||
infos := make([]*querypb.AnalyzerInfo, 0)
|
||||
|
||||
err := validateAnalyzer(collSchema, fieldSchema, &infos)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("field with multi analyzer and analyzer_params", func(t *testing.T) {
|
||||
fieldSchema := createTestFieldSchema("text_field", schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
|
||||
{Key: common.MaxLengthKey, Value: "100"},
|
||||
{Key: "enable_analyzer", Value: "true"},
|
||||
{Key: "multi_analyzer_params", Value: `{"by_field": "lang", "analyzers": {"default": "{}"}}`},
|
||||
{Key: "analyzer_params", Value: `{"type": "standard"}`},
|
||||
})
|
||||
collSchema := createTestCollectionSchemaWithBM25([]*schemapb.FieldSchema{
|
||||
fieldSchema,
|
||||
createTestFieldSchema("lang", schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
|
||||
{Key: common.MaxLengthKey, Value: "10"},
|
||||
}),
|
||||
}, "text_field")
|
||||
infos := make([]*querypb.AnalyzerInfo, 0)
|
||||
|
||||
err := validateAnalyzer(collSchema, fieldSchema, &infos)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("field with valid multi analyzer", func(t *testing.T) {
|
||||
// Create a field with valid multi analyzer
|
||||
fieldSchema := createTestFieldSchema("text_field", schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
|
||||
{Key: common.MaxLengthKey, Value: "100"},
|
||||
{Key: "enable_analyzer", Value: "true"},
|
||||
{Key: "multi_analyzer_params", Value: `{
|
||||
"by_field": "lang",
|
||||
"analyzers": {
|
||||
"default": {"type": "standard"},
|
||||
"english": {"type": "english"}
|
||||
}
|
||||
}`},
|
||||
})
|
||||
collSchema := createTestCollectionSchemaWithBM25([]*schemapb.FieldSchema{
|
||||
fieldSchema,
|
||||
createTestFieldSchema("lang", schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
|
||||
{Key: common.MaxLengthKey, Value: "10"},
|
||||
}),
|
||||
}, "text_field")
|
||||
infos := make([]*querypb.AnalyzerInfo, 0)
|
||||
|
||||
err := validateAnalyzer(collSchema, fieldSchema, &infos)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, infos, 2)
|
||||
|
||||
// Verify analyzer info content
|
||||
analyzerNames := make(map[string]bool)
|
||||
for _, info := range infos {
|
||||
assert.Equal(t, "text_field", info.Field)
|
||||
analyzerNames[info.Name] = true
|
||||
}
|
||||
assert.True(t, analyzerNames["default"])
|
||||
assert.True(t, analyzerNames["english"])
|
||||
})
|
||||
|
||||
t.Run("field with invalid multi analyzer params", func(t *testing.T) {
|
||||
// Create a field with invalid multi analyzer params
|
||||
fieldSchema := createTestFieldSchema("text_field", schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
|
||||
{Key: common.MaxLengthKey, Value: "100"},
|
||||
{Key: "enable_analyzer", Value: "true"},
|
||||
{Key: "multi_analyzer_params", Value: `{"by_field": "non_existent_field", "analyzers": {"default": "{}"}}`},
|
||||
})
|
||||
collSchema := createTestCollectionSchemaWithBM25([]*schemapb.FieldSchema{fieldSchema}, "text_field")
|
||||
infos := make([]*querypb.AnalyzerInfo, 0)
|
||||
|
||||
err := validateAnalyzer(collSchema, fieldSchema, &infos)
|
||||
assert.Error(t, err)
|
||||
})
|
||||
|
||||
t.Run("field with analyzer_params only", func(t *testing.T) {
|
||||
// Create a field with analyzer_params only
|
||||
fieldSchema := createTestFieldSchema("text_field", schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
|
||||
{Key: common.MaxLengthKey, Value: "100"},
|
||||
{Key: "enable_analyzer", Value: "true"},
|
||||
{Key: "analyzer_params", Value: `{"type": "standard"}`},
|
||||
})
|
||||
collSchema := createTestCollectionSchemaWithBM25([]*schemapb.FieldSchema{fieldSchema}, "text_field")
|
||||
infos := make([]*querypb.AnalyzerInfo, 0)
|
||||
|
||||
err := validateAnalyzer(collSchema, fieldSchema, &infos)
|
||||
require.NoError(t, err)
|
||||
require.Len(t, infos, 1)
|
||||
assert.Equal(t, "text_field", infos[0].Field)
|
||||
assert.Equal(t, "", infos[0].Name) // Regular analyzer has no name
|
||||
assert.Equal(t, `{"type": "standard"}`, infos[0].Params)
|
||||
})
|
||||
|
||||
t.Run("field with enable_analyzer but no analyzer_params", func(t *testing.T) {
|
||||
// Create a field with enable_analyzer but no analyzer_params (uses default analyzer)
|
||||
fieldSchema := createTestFieldSchema("text_field", schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
|
||||
{Key: common.MaxLengthKey, Value: "100"},
|
||||
{Key: "enable_match", Value: "true"},
|
||||
{Key: "enable_analyzer", Value: "true"},
|
||||
})
|
||||
collSchema := createTestCollectionSchemaWithBM25([]*schemapb.FieldSchema{fieldSchema}, "text_field")
|
||||
infos := make([]*querypb.AnalyzerInfo, 0)
|
||||
|
||||
err := validateAnalyzer(collSchema, fieldSchema, &infos)
|
||||
assert.NoError(t, err)
|
||||
assert.Len(t, infos, 0) // No analyzer_params, uses default analyzer
|
||||
})
|
||||
}
|
||||
|
||||
@ -398,7 +398,7 @@ func RowBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *schemap
|
||||
}
|
||||
|
||||
for _, field := range collSchema.Fields {
|
||||
if skipFunction && IsBM25FunctionOutputField(field, collSchema) {
|
||||
if skipFunction && typeutil.IsBM25FunctionOutputField(field, collSchema) {
|
||||
continue
|
||||
}
|
||||
|
||||
@ -777,7 +777,7 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
|
||||
}
|
||||
|
||||
handleFieldData := func(field *schemapb.FieldSchema) (FieldData, error) {
|
||||
if IsBM25FunctionOutputField(field, collSchema) {
|
||||
if typeutil.IsBM25FunctionOutputField(field, collSchema) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
@ -1562,25 +1562,6 @@ func (ni NullableInt) IsNull() bool {
|
||||
return ni.Value == nil
|
||||
}
|
||||
|
||||
// TODO: unify the function implementation, storage/utils.go & proxy/util.go
|
||||
func IsBM25FunctionOutputField(field *schemapb.FieldSchema, collSchema *schemapb.CollectionSchema) bool {
|
||||
if !(field.GetIsFunctionOutput() && field.GetDataType() == schemapb.DataType_SparseFloatVector) {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, fSchema := range collSchema.Functions {
|
||||
if fSchema.Type == schemapb.FunctionType_BM25 {
|
||||
if len(fSchema.OutputFieldNames) != 0 && field.Name == fSchema.OutputFieldNames[0] {
|
||||
return true
|
||||
}
|
||||
if len(fSchema.OutputFieldIds) != 0 && field.FieldID == fSchema.OutputFieldIds[0] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func GetDefaultValue(fieldSchema *schemapb.FieldSchema) interface{} {
|
||||
switch fieldSchema.DataType {
|
||||
case schemapb.DataType_Bool:
|
||||
|
||||
@ -2075,7 +2075,7 @@ func TestBM25Checker(t *testing.T) {
|
||||
}
|
||||
|
||||
for _, field := range schema.Fields {
|
||||
isBm25 := IsBM25FunctionOutputField(field, schema)
|
||||
isBm25 := typeutil.IsBM25FunctionOutputField(field, schema)
|
||||
if field.FieldID == 103 {
|
||||
assert.True(t, isBm25)
|
||||
} else {
|
||||
|
||||
@ -193,3 +193,11 @@ func (m *GrpcQueryCoordClient) UpdateLoadConfig(ctx context.Context, req *queryp
|
||||
func (m *GrpcQueryCoordClient) ListLoadedSegments(ctx context.Context, req *querypb.ListLoadedSegmentsRequest, opts ...grpc.CallOption) (*querypb.ListLoadedSegmentsResponse, error) {
|
||||
return &querypb.ListLoadedSegmentsResponse{}, m.Err
|
||||
}
|
||||
|
||||
func (m *GrpcQueryCoordClient) RunAnalyzer(ctx context.Context, req *querypb.RunAnalyzerRequest, opts ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error) {
|
||||
return &milvuspb.RunAnalyzerResponse{}, m.Err
|
||||
}
|
||||
|
||||
func (m *GrpcQueryCoordClient) ValidateAnalyzer(ctx context.Context, req *querypb.ValidateAnalyzerRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
|
||||
return &commonpb.Status{}, m.Err
|
||||
}
|
||||
|
||||
@ -142,6 +142,10 @@ func (m *GrpcQueryNodeClient) RunAnalyzer(ctx context.Context, in *querypb.RunAn
|
||||
return &milvuspb.RunAnalyzerResponse{}, m.Err
|
||||
}
|
||||
|
||||
func (m *GrpcQueryNodeClient) ValidateAnalyzer(ctx context.Context, in *querypb.ValidateAnalyzerRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
|
||||
return &commonpb.Status{}, m.Err
|
||||
}
|
||||
|
||||
func (m *GrpcQueryNodeClient) Close() error {
|
||||
return m.Err
|
||||
}
|
||||
|
||||
@ -164,6 +164,10 @@ func (qn *qnServerWrapper) DropIndex(ctx context.Context, in *querypb.DropIndexR
|
||||
return qn.QueryNode.DropIndex(ctx, in)
|
||||
}
|
||||
|
||||
func (qn *qnServerWrapper) ValidateAnalyzer(ctx context.Context, in *querypb.ValidateAnalyzerRequest, _ ...grpc.CallOption) (*commonpb.Status, error) {
|
||||
return qn.QueryNode.ValidateAnalyzer(ctx, in)
|
||||
}
|
||||
|
||||
func WrapQueryNodeServerAsClient(qn types.QueryNode) types.QueryNodeClient {
|
||||
return &qnServerWrapper{
|
||||
QueryNode: qn,
|
||||
|
||||
@ -100,6 +100,8 @@ service QueryCoord {
|
||||
rpc CheckQueryNodeDistribution(CheckQueryNodeDistributionRequest) returns (common.Status) {}
|
||||
|
||||
rpc UpdateLoadConfig(UpdateLoadConfigRequest) returns (common.Status) {}
|
||||
rpc RunAnalyzer(RunAnalyzerRequest) returns(milvus.RunAnalyzerResponse){}
|
||||
rpc ValidateAnalyzer(ValidateAnalyzerRequest) returns(common.Status){}
|
||||
}
|
||||
|
||||
service QueryNode {
|
||||
@ -173,6 +175,7 @@ service QueryNode {
|
||||
|
||||
rpc RunAnalyzer(RunAnalyzerRequest) returns(milvus.RunAnalyzerResponse){}
|
||||
rpc DropIndex(DropIndexRequest) returns (common.Status) {}
|
||||
rpc ValidateAnalyzer(ValidateAnalyzerRequest) returns(common.Status){}
|
||||
}
|
||||
|
||||
// --------------------QueryCoord grpc request and response proto------------------
|
||||
@ -993,6 +996,19 @@ message RunAnalyzerRequest{
|
||||
|
||||
bool with_detail = 6;
|
||||
bool with_hash = 7;
|
||||
|
||||
string analyzer_params = 8;
|
||||
}
|
||||
|
||||
message AnalyzerInfo{
|
||||
string params = 1;
|
||||
string field = 2;
|
||||
string name = 3;
|
||||
}
|
||||
|
||||
message ValidateAnalyzerRequest{
|
||||
common.MsgBase base = 1;
|
||||
repeated AnalyzerInfo analyzer_infos = 2;
|
||||
}
|
||||
|
||||
message ListLoadedSegmentsRequest {
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@ -59,6 +59,8 @@ const (
|
||||
QueryCoord_TransferChannel_FullMethodName = "/milvus.proto.query.QueryCoord/TransferChannel"
|
||||
QueryCoord_CheckQueryNodeDistribution_FullMethodName = "/milvus.proto.query.QueryCoord/CheckQueryNodeDistribution"
|
||||
QueryCoord_UpdateLoadConfig_FullMethodName = "/milvus.proto.query.QueryCoord/UpdateLoadConfig"
|
||||
QueryCoord_RunAnalyzer_FullMethodName = "/milvus.proto.query.QueryCoord/RunAnalyzer"
|
||||
QueryCoord_ValidateAnalyzer_FullMethodName = "/milvus.proto.query.QueryCoord/ValidateAnalyzer"
|
||||
)
|
||||
|
||||
// QueryCoordClient is the client API for QueryCoord service.
|
||||
@ -106,6 +108,8 @@ type QueryCoordClient interface {
|
||||
TransferChannel(ctx context.Context, in *TransferChannelRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
|
||||
CheckQueryNodeDistribution(ctx context.Context, in *CheckQueryNodeDistributionRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
|
||||
UpdateLoadConfig(ctx context.Context, in *UpdateLoadConfigRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
|
||||
RunAnalyzer(ctx context.Context, in *RunAnalyzerRequest, opts ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error)
|
||||
ValidateAnalyzer(ctx context.Context, in *ValidateAnalyzerRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
|
||||
}
|
||||
|
||||
type queryCoordClient struct {
|
||||
@ -450,6 +454,24 @@ func (c *queryCoordClient) UpdateLoadConfig(ctx context.Context, in *UpdateLoadC
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *queryCoordClient) RunAnalyzer(ctx context.Context, in *RunAnalyzerRequest, opts ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error) {
|
||||
out := new(milvuspb.RunAnalyzerResponse)
|
||||
err := c.cc.Invoke(ctx, QueryCoord_RunAnalyzer_FullMethodName, in, out, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *queryCoordClient) ValidateAnalyzer(ctx context.Context, in *ValidateAnalyzerRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
|
||||
out := new(commonpb.Status)
|
||||
err := c.cc.Invoke(ctx, QueryCoord_ValidateAnalyzer_FullMethodName, in, out, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// QueryCoordServer is the server API for QueryCoord service.
|
||||
// All implementations should embed UnimplementedQueryCoordServer
|
||||
// for forward compatibility
|
||||
@ -495,6 +517,8 @@ type QueryCoordServer interface {
|
||||
TransferChannel(context.Context, *TransferChannelRequest) (*commonpb.Status, error)
|
||||
CheckQueryNodeDistribution(context.Context, *CheckQueryNodeDistributionRequest) (*commonpb.Status, error)
|
||||
UpdateLoadConfig(context.Context, *UpdateLoadConfigRequest) (*commonpb.Status, error)
|
||||
RunAnalyzer(context.Context, *RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error)
|
||||
ValidateAnalyzer(context.Context, *ValidateAnalyzerRequest) (*commonpb.Status, error)
|
||||
}
|
||||
|
||||
// UnimplementedQueryCoordServer should be embedded to have forward compatible implementations.
|
||||
@ -612,6 +636,12 @@ func (UnimplementedQueryCoordServer) CheckQueryNodeDistribution(context.Context,
|
||||
func (UnimplementedQueryCoordServer) UpdateLoadConfig(context.Context, *UpdateLoadConfigRequest) (*commonpb.Status, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method UpdateLoadConfig not implemented")
|
||||
}
|
||||
func (UnimplementedQueryCoordServer) RunAnalyzer(context.Context, *RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method RunAnalyzer not implemented")
|
||||
}
|
||||
func (UnimplementedQueryCoordServer) ValidateAnalyzer(context.Context, *ValidateAnalyzerRequest) (*commonpb.Status, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method ValidateAnalyzer not implemented")
|
||||
}
|
||||
|
||||
// UnsafeQueryCoordServer may be embedded to opt out of forward compatibility for this service.
|
||||
// Use of this interface is not recommended, as added methods to QueryCoordServer will
|
||||
@ -1290,6 +1320,42 @@ func _QueryCoord_UpdateLoadConfig_Handler(srv interface{}, ctx context.Context,
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _QueryCoord_RunAnalyzer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(RunAnalyzerRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(QueryCoordServer).RunAnalyzer(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: QueryCoord_RunAnalyzer_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(QueryCoordServer).RunAnalyzer(ctx, req.(*RunAnalyzerRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _QueryCoord_ValidateAnalyzer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(ValidateAnalyzerRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(QueryCoordServer).ValidateAnalyzer(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: QueryCoord_ValidateAnalyzer_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(QueryCoordServer).ValidateAnalyzer(ctx, req.(*ValidateAnalyzerRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
// QueryCoord_ServiceDesc is the grpc.ServiceDesc for QueryCoord service.
|
||||
// It's only intended for direct use with grpc.RegisterService,
|
||||
// and not to be introspected or modified (even as a copy)
|
||||
@ -1445,6 +1511,14 @@ var QueryCoord_ServiceDesc = grpc.ServiceDesc{
|
||||
MethodName: "UpdateLoadConfig",
|
||||
Handler: _QueryCoord_UpdateLoadConfig_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "RunAnalyzer",
|
||||
Handler: _QueryCoord_RunAnalyzer_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "ValidateAnalyzer",
|
||||
Handler: _QueryCoord_ValidateAnalyzer_Handler,
|
||||
},
|
||||
},
|
||||
Streams: []grpc.StreamDesc{},
|
||||
Metadata: "query_coord.proto",
|
||||
@ -1479,6 +1553,7 @@ const (
|
||||
QueryNode_UpdateSchema_FullMethodName = "/milvus.proto.query.QueryNode/UpdateSchema"
|
||||
QueryNode_RunAnalyzer_FullMethodName = "/milvus.proto.query.QueryNode/RunAnalyzer"
|
||||
QueryNode_DropIndex_FullMethodName = "/milvus.proto.query.QueryNode/DropIndex"
|
||||
QueryNode_ValidateAnalyzer_FullMethodName = "/milvus.proto.query.QueryNode/ValidateAnalyzer"
|
||||
)
|
||||
|
||||
// QueryNodeClient is the client API for QueryNode service.
|
||||
@ -1516,6 +1591,7 @@ type QueryNodeClient interface {
|
||||
UpdateSchema(ctx context.Context, in *UpdateSchemaRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
|
||||
RunAnalyzer(ctx context.Context, in *RunAnalyzerRequest, opts ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error)
|
||||
DropIndex(ctx context.Context, in *DropIndexRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
|
||||
ValidateAnalyzer(ctx context.Context, in *ValidateAnalyzerRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
|
||||
}
|
||||
|
||||
type queryNodeClient struct {
|
||||
@ -1824,6 +1900,15 @@ func (c *queryNodeClient) DropIndex(ctx context.Context, in *DropIndexRequest, o
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (c *queryNodeClient) ValidateAnalyzer(ctx context.Context, in *ValidateAnalyzerRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
|
||||
out := new(commonpb.Status)
|
||||
err := c.cc.Invoke(ctx, QueryNode_ValidateAnalyzer_FullMethodName, in, out, opts...)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// QueryNodeServer is the server API for QueryNode service.
|
||||
// All implementations should embed UnimplementedQueryNodeServer
|
||||
// for forward compatibility
|
||||
@ -1859,6 +1944,7 @@ type QueryNodeServer interface {
|
||||
UpdateSchema(context.Context, *UpdateSchemaRequest) (*commonpb.Status, error)
|
||||
RunAnalyzer(context.Context, *RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error)
|
||||
DropIndex(context.Context, *DropIndexRequest) (*commonpb.Status, error)
|
||||
ValidateAnalyzer(context.Context, *ValidateAnalyzerRequest) (*commonpb.Status, error)
|
||||
}
|
||||
|
||||
// UnimplementedQueryNodeServer should be embedded to have forward compatible implementations.
|
||||
@ -1949,6 +2035,9 @@ func (UnimplementedQueryNodeServer) RunAnalyzer(context.Context, *RunAnalyzerReq
|
||||
func (UnimplementedQueryNodeServer) DropIndex(context.Context, *DropIndexRequest) (*commonpb.Status, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method DropIndex not implemented")
|
||||
}
|
||||
func (UnimplementedQueryNodeServer) ValidateAnalyzer(context.Context, *ValidateAnalyzerRequest) (*commonpb.Status, error) {
|
||||
return nil, status.Errorf(codes.Unimplemented, "method ValidateAnalyzer not implemented")
|
||||
}
|
||||
|
||||
// UnsafeQueryNodeServer may be embedded to opt out of forward compatibility for this service.
|
||||
// Use of this interface is not recommended, as added methods to QueryNodeServer will
|
||||
@ -2471,6 +2560,24 @@ func _QueryNode_DropIndex_Handler(srv interface{}, ctx context.Context, dec func
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
func _QueryNode_ValidateAnalyzer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
|
||||
in := new(ValidateAnalyzerRequest)
|
||||
if err := dec(in); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if interceptor == nil {
|
||||
return srv.(QueryNodeServer).ValidateAnalyzer(ctx, in)
|
||||
}
|
||||
info := &grpc.UnaryServerInfo{
|
||||
Server: srv,
|
||||
FullMethod: QueryNode_ValidateAnalyzer_FullMethodName,
|
||||
}
|
||||
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
|
||||
return srv.(QueryNodeServer).ValidateAnalyzer(ctx, req.(*ValidateAnalyzerRequest))
|
||||
}
|
||||
return interceptor(ctx, in, info, handler)
|
||||
}
|
||||
|
||||
// QueryNode_ServiceDesc is the grpc.ServiceDesc for QueryNode service.
|
||||
// It's only intended for direct use with grpc.RegisterService,
|
||||
// and not to be introspected or modified (even as a copy)
|
||||
@ -2582,6 +2689,10 @@ var QueryNode_ServiceDesc = grpc.ServiceDesc{
|
||||
MethodName: "DropIndex",
|
||||
Handler: _QueryNode_DropIndex_Handler,
|
||||
},
|
||||
{
|
||||
MethodName: "ValidateAnalyzer",
|
||||
Handler: _QueryNode_ValidateAnalyzer_Handler,
|
||||
},
|
||||
},
|
||||
Streams: []grpc.StreamDesc{
|
||||
{
|
||||
|
||||
@ -43,7 +43,7 @@ func Init() error {
|
||||
|
||||
exp, err := CreateTracerExporter(params)
|
||||
if err != nil {
|
||||
log.Warn("Init tracer faield", zap.Error(err))
|
||||
log.Warn("Init tracer failed", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
|
||||
|
||||
@ -199,12 +199,12 @@ func TestGetCollectionIDFromVChannel(t *testing.T) {
|
||||
collectionID := GetCollectionIDFromVChannel(vChannel1)
|
||||
assert.Equal(t, int64(449684528748778322), collectionID)
|
||||
|
||||
invailedVChannel := "06b84fe16780ed1-rootcoord-dm_3_v0"
|
||||
collectionID = GetCollectionIDFromVChannel(invailedVChannel)
|
||||
invaildVChannel := "06b84fe16780ed1-rootcoord-dm_3_v0"
|
||||
collectionID = GetCollectionIDFromVChannel(invaildVChannel)
|
||||
assert.Equal(t, int64(-1), collectionID)
|
||||
|
||||
invailedVChannel = "06b84fe16780ed1-rootcoord-dm_3_-1v0"
|
||||
collectionID = GetCollectionIDFromVChannel(invailedVChannel)
|
||||
invaildVChannel = "06b84fe16780ed1-rootcoord-dm_3_-1v0"
|
||||
collectionID = GetCollectionIDFromVChannel(invaildVChannel)
|
||||
assert.Equal(t, int64(-1), collectionID)
|
||||
}
|
||||
|
||||
|
||||
@ -80,9 +80,6 @@ func (h *FieldSchemaHelper) GetMultiAnalyzerParams() (string, bool) {
|
||||
}
|
||||
|
||||
func (h *FieldSchemaHelper) HasAnalyzerParams() bool {
|
||||
if !IsStringType(h.schema.GetDataType()) {
|
||||
return false
|
||||
}
|
||||
_, err := h.typeParams.Get("analyzer_params")
|
||||
return err == nil
|
||||
}
|
||||
|
||||
@ -2429,3 +2429,30 @@ func GetNeedProcessFunctions(fieldIDs []int64, functions []*schemapb.FunctionSch
|
||||
}
|
||||
return needProcessFunctions, nil
|
||||
}
|
||||
|
||||
func IsBM25FunctionOutputField(field *schemapb.FieldSchema, collSchema *schemapb.CollectionSchema) bool {
|
||||
if !(field.GetIsFunctionOutput() && field.GetDataType() == schemapb.DataType_SparseFloatVector) {
|
||||
return false
|
||||
}
|
||||
|
||||
for _, fSchema := range collSchema.Functions {
|
||||
if fSchema.Type == schemapb.FunctionType_BM25 {
|
||||
if len(fSchema.OutputFieldNames) != 0 && field.Name == fSchema.OutputFieldNames[0] {
|
||||
return true
|
||||
}
|
||||
if len(fSchema.OutputFieldIds) != 0 && field.FieldID == fSchema.OutputFieldIds[0] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func IsBm25FunctionInputField(coll *schemapb.CollectionSchema, field *schemapb.FieldSchema) bool {
|
||||
for _, fn := range coll.GetFunctions() {
|
||||
if fn.GetType() == schemapb.FunctionType_BM25 && field.GetName() == fn.GetInputFieldNames()[0] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
@ -4724,3 +4724,257 @@ func TestUpdateFieldData_GeometryAndTimestamptz(t *testing.T) {
|
||||
assert.Equal(t, []byte{0xFB, 0xFA}, geoData[2])
|
||||
})
|
||||
}
|
||||
|
||||
func TestIsBM25FunctionOutputField(t *testing.T) {
|
||||
schema := &schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{Name: "input_field", DataType: schemapb.DataType_VarChar, TypeParams: []*commonpb.KeyValuePair{{Key: "enable_analyzer", Value: "true"}}},
|
||||
{Name: "output_field", DataType: schemapb.DataType_SparseFloatVector, IsFunctionOutput: true},
|
||||
},
|
||||
Functions: []*schemapb.FunctionSchema{
|
||||
{
|
||||
Name: "bm25_func",
|
||||
Type: schemapb.FunctionType_BM25,
|
||||
InputFieldNames: []string{"input_field"},
|
||||
OutputFieldNames: []string{"output_field"},
|
||||
},
|
||||
},
|
||||
}
|
||||
assert.False(t, IsBM25FunctionOutputField(schema.Fields[0], schema))
|
||||
assert.True(t, IsBM25FunctionOutputField(schema.Fields[1], schema))
|
||||
|
||||
// Test with field IDs instead of names
|
||||
schemaWithIds := &schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{Name: "input_field", FieldID: 100, DataType: schemapb.DataType_VarChar, TypeParams: []*commonpb.KeyValuePair{{Key: "enable_analyzer", Value: "true"}}},
|
||||
{Name: "output_field", FieldID: 101, DataType: schemapb.DataType_SparseFloatVector, IsFunctionOutput: true},
|
||||
},
|
||||
Functions: []*schemapb.FunctionSchema{
|
||||
{
|
||||
Name: "bm25_func",
|
||||
Type: schemapb.FunctionType_BM25,
|
||||
InputFieldIds: []int64{100},
|
||||
OutputFieldIds: []int64{101},
|
||||
},
|
||||
},
|
||||
}
|
||||
assert.False(t, IsBM25FunctionOutputField(schemaWithIds.Fields[0], schemaWithIds))
|
||||
assert.True(t, IsBM25FunctionOutputField(schemaWithIds.Fields[1], schemaWithIds))
|
||||
|
||||
// Test with non-BM25 function
|
||||
nonBM25Schema := &schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{Name: "input_field", DataType: schemapb.DataType_VarChar},
|
||||
{Name: "output_field", DataType: schemapb.DataType_SparseFloatVector, IsFunctionOutput: true},
|
||||
},
|
||||
Functions: []*schemapb.FunctionSchema{
|
||||
{
|
||||
Name: "other_func",
|
||||
Type: schemapb.FunctionType_Unknown,
|
||||
InputFieldNames: []string{"input_field"},
|
||||
OutputFieldNames: []string{"output_field"},
|
||||
},
|
||||
},
|
||||
}
|
||||
assert.False(t, IsBM25FunctionOutputField(nonBM25Schema.Fields[1], nonBM25Schema))
|
||||
|
||||
// Test with non-sparse vector field
|
||||
nonSparseSchema := &schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{Name: "input_field", DataType: schemapb.DataType_VarChar},
|
||||
{Name: "output_field", DataType: schemapb.DataType_FloatVector, IsFunctionOutput: true},
|
||||
},
|
||||
Functions: []*schemapb.FunctionSchema{
|
||||
{
|
||||
Name: "bm25_func",
|
||||
Type: schemapb.FunctionType_BM25,
|
||||
InputFieldNames: []string{"input_field"},
|
||||
OutputFieldNames: []string{"output_field"},
|
||||
},
|
||||
},
|
||||
}
|
||||
assert.False(t, IsBM25FunctionOutputField(nonSparseSchema.Fields[1], nonSparseSchema))
|
||||
|
||||
// Test with field not marked as function output
|
||||
nonFunctionOutputSchema := &schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{Name: "input_field", DataType: schemapb.DataType_VarChar},
|
||||
{Name: "output_field", DataType: schemapb.DataType_SparseFloatVector, IsFunctionOutput: false},
|
||||
},
|
||||
Functions: []*schemapb.FunctionSchema{
|
||||
{
|
||||
Name: "bm25_func",
|
||||
Type: schemapb.FunctionType_BM25,
|
||||
InputFieldNames: []string{"input_field"},
|
||||
OutputFieldNames: []string{"output_field"},
|
||||
},
|
||||
},
|
||||
}
|
||||
assert.False(t, IsBM25FunctionOutputField(nonFunctionOutputSchema.Fields[1], nonFunctionOutputSchema))
|
||||
}
|
||||
|
||||
func TestIsBm25FunctionInputField(t *testing.T) {
|
||||
schema := &schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{Name: "input_field", DataType: schemapb.DataType_VarChar, TypeParams: []*commonpb.KeyValuePair{{Key: "enable_analyzer", Value: "true"}}},
|
||||
{Name: "output_field", DataType: schemapb.DataType_SparseFloatVector, IsFunctionOutput: true},
|
||||
},
|
||||
Functions: []*schemapb.FunctionSchema{
|
||||
{
|
||||
Name: "bm25_func",
|
||||
Type: schemapb.FunctionType_BM25,
|
||||
InputFieldNames: []string{"input_field"},
|
||||
OutputFieldNames: []string{"output_field"},
|
||||
},
|
||||
},
|
||||
}
|
||||
assert.True(t, IsBm25FunctionInputField(schema, schema.Fields[0]))
|
||||
assert.False(t, IsBm25FunctionInputField(schema, schema.Fields[1]))
|
||||
|
||||
// Test with multiple functions, only one is BM25
|
||||
multipleSchema := &schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{Name: "input_field1", DataType: schemapb.DataType_VarChar},
|
||||
{Name: "input_field2", DataType: schemapb.DataType_VarChar},
|
||||
{Name: "output_field1", DataType: schemapb.DataType_SparseFloatVector, IsFunctionOutput: true},
|
||||
{Name: "output_field2", DataType: schemapb.DataType_FloatVector, IsFunctionOutput: true},
|
||||
},
|
||||
Functions: []*schemapb.FunctionSchema{
|
||||
{
|
||||
Name: "bm25_func",
|
||||
Type: schemapb.FunctionType_BM25,
|
||||
InputFieldNames: []string{"input_field1"},
|
||||
OutputFieldNames: []string{"output_field1"},
|
||||
},
|
||||
{
|
||||
Name: "other_func",
|
||||
Type: schemapb.FunctionType_Unknown,
|
||||
InputFieldNames: []string{"input_field2"},
|
||||
OutputFieldNames: []string{"output_field2"},
|
||||
},
|
||||
},
|
||||
}
|
||||
assert.True(t, IsBm25FunctionInputField(multipleSchema, multipleSchema.Fields[0]))
|
||||
assert.False(t, IsBm25FunctionInputField(multipleSchema, multipleSchema.Fields[1]))
|
||||
assert.False(t, IsBm25FunctionInputField(multipleSchema, multipleSchema.Fields[2]))
|
||||
assert.False(t, IsBm25FunctionInputField(multipleSchema, multipleSchema.Fields[3]))
|
||||
|
||||
// Test with no BM25 functions
|
||||
noBM25Schema := &schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{Name: "input_field", DataType: schemapb.DataType_VarChar},
|
||||
{Name: "output_field", DataType: schemapb.DataType_FloatVector, IsFunctionOutput: true},
|
||||
},
|
||||
Functions: []*schemapb.FunctionSchema{
|
||||
{
|
||||
Name: "other_func",
|
||||
Type: schemapb.FunctionType_Unknown,
|
||||
InputFieldNames: []string{"input_field"},
|
||||
OutputFieldNames: []string{"output_field"},
|
||||
},
|
||||
},
|
||||
}
|
||||
assert.False(t, IsBm25FunctionInputField(noBM25Schema, noBM25Schema.Fields[0]))
|
||||
|
||||
// Test with empty functions
|
||||
emptySchema := &schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{Name: "input_field", DataType: schemapb.DataType_VarChar},
|
||||
},
|
||||
Functions: []*schemapb.FunctionSchema{},
|
||||
}
|
||||
assert.False(t, IsBm25FunctionInputField(emptySchema, emptySchema.Fields[0]))
|
||||
|
||||
// Test with nil functions
|
||||
nilSchema := &schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{Name: "input_field", DataType: schemapb.DataType_VarChar},
|
||||
},
|
||||
Functions: nil,
|
||||
}
|
||||
assert.False(t, IsBm25FunctionInputField(nilSchema, nilSchema.Fields[0]))
|
||||
}
|
||||
|
||||
func TestSchemaHelper_GetFunctionByOutputField(t *testing.T) {
|
||||
schema := &schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{Name: "input_field", FieldID: 100, DataType: schemapb.DataType_VarChar},
|
||||
{Name: "output_field", FieldID: 101, DataType: schemapb.DataType_SparseFloatVector, IsFunctionOutput: true},
|
||||
{Name: "regular_field", FieldID: 102, DataType: schemapb.DataType_Int64},
|
||||
},
|
||||
Functions: []*schemapb.FunctionSchema{
|
||||
{
|
||||
Name: "bm25_func",
|
||||
Type: schemapb.FunctionType_BM25,
|
||||
InputFieldIds: []int64{100},
|
||||
OutputFieldIds: []int64{101},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
helper, err := CreateSchemaHelper(schema)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Test getting function by output field
|
||||
function, err := helper.GetFunctionByOutputField(schema.Fields[1])
|
||||
assert.NoError(t, err)
|
||||
assert.Equal(t, "bm25_func", function.Name)
|
||||
assert.Equal(t, schemapb.FunctionType_BM25, function.Type)
|
||||
|
||||
// Test with non-function output field
|
||||
_, err = helper.GetFunctionByOutputField(schema.Fields[0])
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "function not exist")
|
||||
|
||||
// Test with regular field
|
||||
_, err = helper.GetFunctionByOutputField(schema.Fields[2])
|
||||
assert.Error(t, err)
|
||||
assert.Contains(t, err.Error(), "function not exist")
|
||||
}
|
||||
|
||||
func TestSchemaHelper_CanRetrieveRawFieldData(t *testing.T) {
|
||||
schema := &schemapb.CollectionSchema{
|
||||
Fields: []*schemapb.FieldSchema{
|
||||
{Name: "input_field", FieldID: 100, DataType: schemapb.DataType_VarChar},
|
||||
{Name: "bm25_output", FieldID: 101, DataType: schemapb.DataType_SparseFloatVector, IsFunctionOutput: true},
|
||||
{Name: "other_output", FieldID: 102, DataType: schemapb.DataType_FloatVector, IsFunctionOutput: true},
|
||||
{Name: "regular_field", FieldID: 103, DataType: schemapb.DataType_Int64},
|
||||
},
|
||||
Functions: []*schemapb.FunctionSchema{
|
||||
{
|
||||
Name: "bm25_func",
|
||||
Type: schemapb.FunctionType_BM25,
|
||||
InputFieldIds: []int64{100},
|
||||
OutputFieldIds: []int64{101},
|
||||
},
|
||||
{
|
||||
Name: "other_func",
|
||||
Type: schemapb.FunctionType_Unknown,
|
||||
InputFieldIds: []int64{100},
|
||||
OutputFieldIds: []int64{102},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
helper, err := CreateSchemaHelper(schema)
|
||||
assert.NoError(t, err)
|
||||
|
||||
// Regular field should be retrievable
|
||||
assert.True(t, helper.CanRetrieveRawFieldData(schema.Fields[0]))
|
||||
assert.True(t, helper.CanRetrieveRawFieldData(schema.Fields[3]))
|
||||
|
||||
// BM25 function output field should NOT be retrievable
|
||||
assert.False(t, helper.CanRetrieveRawFieldData(schema.Fields[1]))
|
||||
|
||||
// Other function output field should be retrievable
|
||||
assert.True(t, helper.CanRetrieveRawFieldData(schema.Fields[2]))
|
||||
|
||||
// Test with field that has IsFunctionOutput=true but no corresponding function
|
||||
orphanField := &schemapb.FieldSchema{
|
||||
Name: "orphan_field",
|
||||
FieldID: 104,
|
||||
DataType: schemapb.DataType_FloatVector,
|
||||
IsFunctionOutput: true,
|
||||
}
|
||||
assert.False(t, helper.CanRetrieveRawFieldData(orphanField))
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user