enhance: forbid build analyzer at proxy (#44067)

relate: https://github.com/milvus-io/milvus/issues/43687
We used to run the temporary analyzer and validate analyzer on the
proxy, but the proxy should not be a computation-heavy node. This PR
move all analyzer calculations to the streaming node.

---------

Signed-off-by: aoiasd <zhicheng.yue@zilliz.com>
This commit is contained in:
aoiasd 2025-10-23 10:58:12 +08:00 committed by GitHub
parent 20dcb45b3d
commit cfeb095ad7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
48 changed files with 3126 additions and 1043 deletions

View File

@ -478,7 +478,7 @@ func (mr *MilvusRoles) Run() {
exp, err := tracer.CreateTracerExporter(params) exp, err := tracer.CreateTracerExporter(params)
if err != nil { if err != nil {
log.Warn("Init tracer faield", zap.Error(err)) log.Warn("Init tracer failed", zap.Error(err))
return return
} }

View File

@ -1193,6 +1193,14 @@ func (s *mixCoordImpl) ListLoadedSegments(ctx context.Context, req *querypb.List
return s.queryCoordServer.ListLoadedSegments(ctx, req) return s.queryCoordServer.ListLoadedSegments(ctx, req)
} }
func (s *mixCoordImpl) RunAnalyzer(ctx context.Context, req *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) {
return s.queryCoordServer.RunAnalyzer(ctx, req)
}
func (s *mixCoordImpl) ValidateAnalyzer(ctx context.Context, req *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error) {
return s.queryCoordServer.ValidateAnalyzer(ctx, req)
}
func (s *mixCoordImpl) FlushAll(ctx context.Context, req *datapb.FlushAllRequest) (*datapb.FlushAllResponse, error) { func (s *mixCoordImpl) FlushAll(ctx context.Context, req *datapb.FlushAllRequest) (*datapb.FlushAllResponse, error) {
return s.datacoordServer.FlushAll(ctx, req) return s.datacoordServer.FlushAll(ctx, req)
} }

View File

@ -922,6 +922,14 @@ func (s *mockMixCoord) ListFileResources(ctx context.Context, req *milvuspb.List
panic("implement me") panic("implement me")
} }
func (s *mockMixCoord) RunAnalyzer(ctx context.Context, req *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) {
panic("implement me")
}
func (s *mockMixCoord) ValidateAnalyzer(ctx context.Context, req *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error) {
panic("implement me")
}
type mockHandler struct { type mockHandler struct {
meta *meta meta *meta
} }

View File

@ -1893,3 +1893,25 @@ func (c *Client) ListFileResources(ctx context.Context, req *milvuspb.ListFileRe
return client.ListFileResources(ctx, req) return client.ListFileResources(ctx, req)
}) })
} }
func (c *Client) RunAnalyzer(ctx context.Context, req *querypb.RunAnalyzerRequest, opts ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error) {
req = typeutil.Clone(req)
commonpbutil.UpdateMsgBase(
req.GetBase(),
commonpbutil.FillMsgBaseFromClient(paramtable.GetNodeID(), commonpbutil.WithTargetID(c.grpcClient.GetNodeID())),
)
return wrapGrpcCall(ctx, c, func(client MixCoordClient) (*milvuspb.RunAnalyzerResponse, error) {
return client.RunAnalyzer(ctx, req)
})
}
func (c *Client) ValidateAnalyzer(ctx context.Context, req *querypb.ValidateAnalyzerRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
req = typeutil.Clone(req)
commonpbutil.UpdateMsgBase(
req.GetBase(),
commonpbutil.FillMsgBaseFromClient(paramtable.GetNodeID(), commonpbutil.WithTargetID(c.grpcClient.GetNodeID())),
)
return wrapGrpcCall(ctx, c, func(client MixCoordClient) (*commonpb.Status, error) {
return client.ValidateAnalyzer(ctx, req)
})
}

View File

@ -384,6 +384,12 @@ func Test_NewClient(t *testing.T) {
r40, err := client.CheckBalanceStatus(ctx, nil) r40, err := client.CheckBalanceStatus(ctx, nil)
retCheck(retNotNil, r40, err) retCheck(retNotNil, r40, err)
r41, err := client.RunAnalyzer(ctx, nil)
retCheck(retNotNil, r41, err)
r42, err := client.ValidateAnalyzer(ctx, nil)
retCheck(retNotNil, r42, err)
} }
client.(*Client).grpcClient = &mock.GRPCClientBase[MixCoordClient]{ client.(*Client).grpcClient = &mock.GRPCClientBase[MixCoordClient]{

View File

@ -920,6 +920,14 @@ func (s *Server) ListLoadedSegments(ctx context.Context, req *querypb.ListLoaded
return s.mixCoord.ListLoadedSegments(ctx, req) return s.mixCoord.ListLoadedSegments(ctx, req)
} }
func (s *Server) RunAnalyzer(ctx context.Context, req *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) {
return s.mixCoord.RunAnalyzer(ctx, req)
}
func (s *Server) ValidateAnalyzer(ctx context.Context, req *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error) {
return s.mixCoord.ValidateAnalyzer(ctx, req)
}
// AddFileResource add file resource // AddFileResource add file resource
func (s *Server) AddFileResource(ctx context.Context, req *milvuspb.AddFileResourceRequest) (*commonpb.Status, error) { func (s *Server) AddFileResource(ctx context.Context, req *milvuspb.AddFileResourceRequest) (*commonpb.Status, error) {
return s.mixCoord.AddFileResource(ctx, req) return s.mixCoord.AddFileResource(ctx, req)

View File

@ -760,4 +760,22 @@ func Test_NewServer(t *testing.T) {
assert.NoError(t, err) assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode()) assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
}) })
t.Run("RunAnalyzer", func(t *testing.T) {
req := &querypb.RunAnalyzerRequest{}
mockMixCoord.EXPECT().RunAnalyzer(mock.Anything, req).Return(&milvuspb.RunAnalyzerResponse{
Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success},
}, nil)
resp, err := server.RunAnalyzer(ctx, req)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
})
t.Run("ValidateAnalyzer", func(t *testing.T) {
req := &querypb.ValidateAnalyzerRequest{}
mockMixCoord.EXPECT().ValidateAnalyzer(mock.Anything, req).Return(&commonpb.Status{ErrorCode: commonpb.ErrorCode_Success}, nil)
resp, err := server.ValidateAnalyzer(ctx, req)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.ErrorCode)
})
} }

View File

@ -688,6 +688,14 @@ func Test_NewServer(t *testing.T) {
assert.Nil(t, err) assert.Nil(t, err)
}) })
t.Run("RunAnalyzer", func(t *testing.T) {
mockProxy.EXPECT().RunAnalyzer(mock.Anything, mock.Anything).Return(&milvuspb.RunAnalyzerResponse{
Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success},
}, nil)
_, err := server.RunAnalyzer(ctx, &milvuspb.RunAnalyzerRequest{})
assert.NoError(t, err)
})
t.Run("Run with different config", func(t *testing.T) { t.Run("Run with different config", func(t *testing.T) {
mockProxy.EXPECT().Init().Return(nil) mockProxy.EXPECT().Init().Return(nil)
mockProxy.EXPECT().Start().Return(nil) mockProxy.EXPECT().Start().Return(nil)

View File

@ -394,3 +394,14 @@ func (c *Client) DropIndex(ctx context.Context, req *querypb.DropIndexRequest, _
return client.DropIndex(ctx, req) return client.DropIndex(ctx, req)
}) })
} }
func (c *Client) ValidateAnalyzer(ctx context.Context, req *querypb.ValidateAnalyzerRequest, _ ...grpc.CallOption) (*commonpb.Status, error) {
req = typeutil.Clone(req)
commonpbutil.UpdateMsgBase(
req.GetBase(),
commonpbutil.FillMsgBaseFromClient(c.nodeID),
)
return wrapGrpcCall(ctx, c, func(client querypb.QueryNodeClient) (*commonpb.Status, error) {
return client.ValidateAnalyzer(ctx, req)
})
}

View File

@ -111,6 +111,12 @@ func Test_NewClient(t *testing.T) {
r21, err := client.DeleteBatch(ctx, nil) r21, err := client.DeleteBatch(ctx, nil)
retCheck(retNotNil, r21, err) retCheck(retNotNil, r21, err)
r22, err := client.RunAnalyzer(ctx, nil)
retCheck(retNotNil, r22, err)
r23, err := client.ValidateAnalyzer(ctx, nil)
retCheck(retNotNil, r23, err)
// stream rpc // stream rpc
client, err := client.QueryStream(ctx, nil) client, err := client.QueryStream(ctx, nil)
retCheck(retNotNil, client, err) retCheck(retNotNil, client, err)

View File

@ -406,3 +406,7 @@ func (s *Server) RunAnalyzer(ctx context.Context, req *querypb.RunAnalyzerReques
func (s *Server) DropIndex(ctx context.Context, req *querypb.DropIndexRequest) (*commonpb.Status, error) { func (s *Server) DropIndex(ctx context.Context, req *querypb.DropIndexRequest) (*commonpb.Status, error) {
return s.querynode.DropIndex(ctx, req) return s.querynode.DropIndex(ctx, req)
} }
func (s *Server) ValidateAnalyzer(ctx context.Context, req *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error) {
return s.querynode.ValidateAnalyzer(ctx, req)
}

View File

@ -280,6 +280,24 @@ func Test_NewServer(t *testing.T) {
assert.NoError(t, merr.CheckRPCCall(resp, err)) assert.NoError(t, merr.CheckRPCCall(resp, err))
}) })
t.Run("RunAnalyzer", func(t *testing.T) {
mockQN.EXPECT().RunAnalyzer(mock.Anything, mock.Anything).Return(&milvuspb.RunAnalyzerResponse{
Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success},
}, nil)
req := &querypb.RunAnalyzerRequest{}
resp, err := server.RunAnalyzer(ctx, req)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.GetStatus().GetErrorCode())
})
t.Run("ValidateAnalyzer", func(t *testing.T) {
mockQN.EXPECT().ValidateAnalyzer(mock.Anything, mock.Anything).Return(&commonpb.Status{ErrorCode: commonpb.ErrorCode_Success}, nil)
req := &querypb.ValidateAnalyzerRequest{}
resp, err := server.ValidateAnalyzer(ctx, req)
assert.NoError(t, err)
assert.Equal(t, commonpb.ErrorCode_Success, resp.ErrorCode)
})
err = server.Stop() err = server.Stop()
assert.NoError(t, err) assert.NoError(t, err)
} }

View File

@ -633,7 +633,7 @@ func (ss *SuffixSnapshot) startBackgroundGC(ctx context.Context) {
func (ss *SuffixSnapshot) getOriginalKey(snapshotKey string) (string, error) { func (ss *SuffixSnapshot) getOriginalKey(snapshotKey string) (string, error) {
if !strings.HasPrefix(snapshotKey, ss.snapshotPrefix) { if !strings.HasPrefix(snapshotKey, ss.snapshotPrefix) {
return "", fmt.Errorf("get original key failed, invailed snapshot key:%s", snapshotKey) return "", fmt.Errorf("get original key failed, invaild snapshot key:%s", snapshotKey)
} }
// collect keys that parent node is snapshot node if the corresponding the latest ts is expired. // collect keys that parent node is snapshot node if the corresponding the latest ts is expired.
idx := strings.LastIndex(snapshotKey, ss.separator) idx := strings.LastIndex(snapshotKey, ss.separator)

View File

@ -6973,6 +6973,65 @@ func (_c *MixCoord_ResumeNode_Call) RunAndReturn(run func(context.Context, *quer
return _c return _c
} }
// RunAnalyzer provides a mock function with given fields: _a0, _a1
func (_m *MixCoord) RunAnalyzer(_a0 context.Context, _a1 *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) {
ret := _m.Called(_a0, _a1)
if len(ret) == 0 {
panic("no return value specified for RunAnalyzer")
}
var r0 *milvuspb.RunAnalyzerResponse
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error)); ok {
return rf(_a0, _a1)
}
if rf, ok := ret.Get(0).(func(context.Context, *querypb.RunAnalyzerRequest) *milvuspb.RunAnalyzerResponse); ok {
r0 = rf(_a0, _a1)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*milvuspb.RunAnalyzerResponse)
}
}
if rf, ok := ret.Get(1).(func(context.Context, *querypb.RunAnalyzerRequest) error); ok {
r1 = rf(_a0, _a1)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MixCoord_RunAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RunAnalyzer'
type MixCoord_RunAnalyzer_Call struct {
*mock.Call
}
// RunAnalyzer is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *querypb.RunAnalyzerRequest
func (_e *MixCoord_Expecter) RunAnalyzer(_a0 interface{}, _a1 interface{}) *MixCoord_RunAnalyzer_Call {
return &MixCoord_RunAnalyzer_Call{Call: _e.mock.On("RunAnalyzer", _a0, _a1)}
}
func (_c *MixCoord_RunAnalyzer_Call) Run(run func(_a0 context.Context, _a1 *querypb.RunAnalyzerRequest)) *MixCoord_RunAnalyzer_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(*querypb.RunAnalyzerRequest))
})
return _c
}
func (_c *MixCoord_RunAnalyzer_Call) Return(_a0 *milvuspb.RunAnalyzerResponse, _a1 error) *MixCoord_RunAnalyzer_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MixCoord_RunAnalyzer_Call) RunAndReturn(run func(context.Context, *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error)) *MixCoord_RunAnalyzer_Call {
_c.Call.Return(run)
return _c
}
// SaveBinlogPaths provides a mock function with given fields: _a0, _a1 // SaveBinlogPaths provides a mock function with given fields: _a0, _a1
func (_m *MixCoord) SaveBinlogPaths(_a0 context.Context, _a1 *datapb.SaveBinlogPathsRequest) (*commonpb.Status, error) { func (_m *MixCoord) SaveBinlogPaths(_a0 context.Context, _a1 *datapb.SaveBinlogPathsRequest) (*commonpb.Status, error) {
ret := _m.Called(_a0, _a1) ret := _m.Called(_a0, _a1)
@ -8762,6 +8821,65 @@ func (_c *MixCoord_UpdateStateCode_Call) RunAndReturn(run func(commonpb.StateCod
return _c return _c
} }
// ValidateAnalyzer provides a mock function with given fields: _a0, _a1
func (_m *MixCoord) ValidateAnalyzer(_a0 context.Context, _a1 *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error) {
ret := _m.Called(_a0, _a1)
if len(ret) == 0 {
panic("no return value specified for ValidateAnalyzer")
}
var r0 *commonpb.Status
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error)); ok {
return rf(_a0, _a1)
}
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest) *commonpb.Status); ok {
r0 = rf(_a0, _a1)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*commonpb.Status)
}
}
if rf, ok := ret.Get(1).(func(context.Context, *querypb.ValidateAnalyzerRequest) error); ok {
r1 = rf(_a0, _a1)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MixCoord_ValidateAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ValidateAnalyzer'
type MixCoord_ValidateAnalyzer_Call struct {
*mock.Call
}
// ValidateAnalyzer is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *querypb.ValidateAnalyzerRequest
func (_e *MixCoord_Expecter) ValidateAnalyzer(_a0 interface{}, _a1 interface{}) *MixCoord_ValidateAnalyzer_Call {
return &MixCoord_ValidateAnalyzer_Call{Call: _e.mock.On("ValidateAnalyzer", _a0, _a1)}
}
func (_c *MixCoord_ValidateAnalyzer_Call) Run(run func(_a0 context.Context, _a1 *querypb.ValidateAnalyzerRequest)) *MixCoord_ValidateAnalyzer_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(*querypb.ValidateAnalyzerRequest))
})
return _c
}
func (_c *MixCoord_ValidateAnalyzer_Call) Return(_a0 *commonpb.Status, _a1 error) *MixCoord_ValidateAnalyzer_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MixCoord_ValidateAnalyzer_Call) RunAndReturn(run func(context.Context, *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error)) *MixCoord_ValidateAnalyzer_Call {
_c.Call.Return(run)
return _c
}
// WatchChannels provides a mock function with given fields: _a0, _a1 // WatchChannels provides a mock function with given fields: _a0, _a1
func (_m *MixCoord) WatchChannels(_a0 context.Context, _a1 *datapb.WatchChannelsRequest) (*datapb.WatchChannelsResponse, error) { func (_m *MixCoord) WatchChannels(_a0 context.Context, _a1 *datapb.WatchChannelsRequest) (*datapb.WatchChannelsResponse, error) {
ret := _m.Called(_a0, _a1) ret := _m.Called(_a0, _a1)

View File

@ -8372,6 +8372,80 @@ func (_c *MockMixCoordClient_ResumeNode_Call) RunAndReturn(run func(context.Cont
return _c return _c
} }
// RunAnalyzer provides a mock function with given fields: ctx, in, opts
func (_m *MockMixCoordClient) RunAnalyzer(ctx context.Context, in *querypb.RunAnalyzerRequest, opts ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error) {
_va := make([]interface{}, len(opts))
for _i := range opts {
_va[_i] = opts[_i]
}
var _ca []interface{}
_ca = append(_ca, ctx, in)
_ca = append(_ca, _va...)
ret := _m.Called(_ca...)
if len(ret) == 0 {
panic("no return value specified for RunAnalyzer")
}
var r0 *milvuspb.RunAnalyzerResponse
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *querypb.RunAnalyzerRequest, ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error)); ok {
return rf(ctx, in, opts...)
}
if rf, ok := ret.Get(0).(func(context.Context, *querypb.RunAnalyzerRequest, ...grpc.CallOption) *milvuspb.RunAnalyzerResponse); ok {
r0 = rf(ctx, in, opts...)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*milvuspb.RunAnalyzerResponse)
}
}
if rf, ok := ret.Get(1).(func(context.Context, *querypb.RunAnalyzerRequest, ...grpc.CallOption) error); ok {
r1 = rf(ctx, in, opts...)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockMixCoordClient_RunAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RunAnalyzer'
type MockMixCoordClient_RunAnalyzer_Call struct {
*mock.Call
}
// RunAnalyzer is a helper method to define mock.On call
// - ctx context.Context
// - in *querypb.RunAnalyzerRequest
// - opts ...grpc.CallOption
func (_e *MockMixCoordClient_Expecter) RunAnalyzer(ctx interface{}, in interface{}, opts ...interface{}) *MockMixCoordClient_RunAnalyzer_Call {
return &MockMixCoordClient_RunAnalyzer_Call{Call: _e.mock.On("RunAnalyzer",
append([]interface{}{ctx, in}, opts...)...)}
}
func (_c *MockMixCoordClient_RunAnalyzer_Call) Run(run func(ctx context.Context, in *querypb.RunAnalyzerRequest, opts ...grpc.CallOption)) *MockMixCoordClient_RunAnalyzer_Call {
_c.Call.Run(func(args mock.Arguments) {
variadicArgs := make([]grpc.CallOption, len(args)-2)
for i, a := range args[2:] {
if a != nil {
variadicArgs[i] = a.(grpc.CallOption)
}
}
run(args[0].(context.Context), args[1].(*querypb.RunAnalyzerRequest), variadicArgs...)
})
return _c
}
func (_c *MockMixCoordClient_RunAnalyzer_Call) Return(_a0 *milvuspb.RunAnalyzerResponse, _a1 error) *MockMixCoordClient_RunAnalyzer_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockMixCoordClient_RunAnalyzer_Call) RunAndReturn(run func(context.Context, *querypb.RunAnalyzerRequest, ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error)) *MockMixCoordClient_RunAnalyzer_Call {
_c.Call.Return(run)
return _c
}
// SaveBinlogPaths provides a mock function with given fields: ctx, in, opts // SaveBinlogPaths provides a mock function with given fields: ctx, in, opts
func (_m *MockMixCoordClient) SaveBinlogPaths(ctx context.Context, in *datapb.SaveBinlogPathsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) { func (_m *MockMixCoordClient) SaveBinlogPaths(ctx context.Context, in *datapb.SaveBinlogPathsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
_va := make([]interface{}, len(opts)) _va := make([]interface{}, len(opts))
@ -10296,6 +10370,80 @@ func (_c *MockMixCoordClient_UpdateSegmentStatistics_Call) RunAndReturn(run func
return _c return _c
} }
// ValidateAnalyzer provides a mock function with given fields: ctx, in, opts
func (_m *MockMixCoordClient) ValidateAnalyzer(ctx context.Context, in *querypb.ValidateAnalyzerRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
_va := make([]interface{}, len(opts))
for _i := range opts {
_va[_i] = opts[_i]
}
var _ca []interface{}
_ca = append(_ca, ctx, in)
_ca = append(_ca, _va...)
ret := _m.Called(_ca...)
if len(ret) == 0 {
panic("no return value specified for ValidateAnalyzer")
}
var r0 *commonpb.Status
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest, ...grpc.CallOption) (*commonpb.Status, error)); ok {
return rf(ctx, in, opts...)
}
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest, ...grpc.CallOption) *commonpb.Status); ok {
r0 = rf(ctx, in, opts...)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*commonpb.Status)
}
}
if rf, ok := ret.Get(1).(func(context.Context, *querypb.ValidateAnalyzerRequest, ...grpc.CallOption) error); ok {
r1 = rf(ctx, in, opts...)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockMixCoordClient_ValidateAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ValidateAnalyzer'
type MockMixCoordClient_ValidateAnalyzer_Call struct {
*mock.Call
}
// ValidateAnalyzer is a helper method to define mock.On call
// - ctx context.Context
// - in *querypb.ValidateAnalyzerRequest
// - opts ...grpc.CallOption
func (_e *MockMixCoordClient_Expecter) ValidateAnalyzer(ctx interface{}, in interface{}, opts ...interface{}) *MockMixCoordClient_ValidateAnalyzer_Call {
return &MockMixCoordClient_ValidateAnalyzer_Call{Call: _e.mock.On("ValidateAnalyzer",
append([]interface{}{ctx, in}, opts...)...)}
}
func (_c *MockMixCoordClient_ValidateAnalyzer_Call) Run(run func(ctx context.Context, in *querypb.ValidateAnalyzerRequest, opts ...grpc.CallOption)) *MockMixCoordClient_ValidateAnalyzer_Call {
_c.Call.Run(func(args mock.Arguments) {
variadicArgs := make([]grpc.CallOption, len(args)-2)
for i, a := range args[2:] {
if a != nil {
variadicArgs[i] = a.(grpc.CallOption)
}
}
run(args[0].(context.Context), args[1].(*querypb.ValidateAnalyzerRequest), variadicArgs...)
})
return _c
}
func (_c *MockMixCoordClient_ValidateAnalyzer_Call) Return(_a0 *commonpb.Status, _a1 error) *MockMixCoordClient_ValidateAnalyzer_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockMixCoordClient_ValidateAnalyzer_Call) RunAndReturn(run func(context.Context, *querypb.ValidateAnalyzerRequest, ...grpc.CallOption) (*commonpb.Status, error)) *MockMixCoordClient_ValidateAnalyzer_Call {
_c.Call.Return(run)
return _c
}
// WatchChannels provides a mock function with given fields: ctx, in, opts // WatchChannels provides a mock function with given fields: ctx, in, opts
func (_m *MockMixCoordClient) WatchChannels(ctx context.Context, in *datapb.WatchChannelsRequest, opts ...grpc.CallOption) (*datapb.WatchChannelsResponse, error) { func (_m *MockMixCoordClient) WatchChannels(ctx context.Context, in *datapb.WatchChannelsRequest, opts ...grpc.CallOption) (*datapb.WatchChannelsResponse, error) {
_va := make([]interface{}, len(opts)) _va := make([]interface{}, len(opts))

View File

@ -1599,6 +1599,65 @@ func (_c *MockQueryCoord_ResumeNode_Call) RunAndReturn(run func(context.Context,
return _c return _c
} }
// RunAnalyzer provides a mock function with given fields: _a0, _a1
func (_m *MockQueryCoord) RunAnalyzer(_a0 context.Context, _a1 *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) {
ret := _m.Called(_a0, _a1)
if len(ret) == 0 {
panic("no return value specified for RunAnalyzer")
}
var r0 *milvuspb.RunAnalyzerResponse
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error)); ok {
return rf(_a0, _a1)
}
if rf, ok := ret.Get(0).(func(context.Context, *querypb.RunAnalyzerRequest) *milvuspb.RunAnalyzerResponse); ok {
r0 = rf(_a0, _a1)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*milvuspb.RunAnalyzerResponse)
}
}
if rf, ok := ret.Get(1).(func(context.Context, *querypb.RunAnalyzerRequest) error); ok {
r1 = rf(_a0, _a1)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockQueryCoord_RunAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RunAnalyzer'
type MockQueryCoord_RunAnalyzer_Call struct {
*mock.Call
}
// RunAnalyzer is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *querypb.RunAnalyzerRequest
func (_e *MockQueryCoord_Expecter) RunAnalyzer(_a0 interface{}, _a1 interface{}) *MockQueryCoord_RunAnalyzer_Call {
return &MockQueryCoord_RunAnalyzer_Call{Call: _e.mock.On("RunAnalyzer", _a0, _a1)}
}
func (_c *MockQueryCoord_RunAnalyzer_Call) Run(run func(_a0 context.Context, _a1 *querypb.RunAnalyzerRequest)) *MockQueryCoord_RunAnalyzer_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(*querypb.RunAnalyzerRequest))
})
return _c
}
func (_c *MockQueryCoord_RunAnalyzer_Call) Return(_a0 *milvuspb.RunAnalyzerResponse, _a1 error) *MockQueryCoord_RunAnalyzer_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockQueryCoord_RunAnalyzer_Call) RunAndReturn(run func(context.Context, *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error)) *MockQueryCoord_RunAnalyzer_Call {
_c.Call.Return(run)
return _c
}
// SetAddress provides a mock function with given fields: address // SetAddress provides a mock function with given fields: address
func (_m *MockQueryCoord) SetAddress(address string) { func (_m *MockQueryCoord) SetAddress(address string) {
_m.Called(address) _m.Called(address)
@ -2595,6 +2654,65 @@ func (_c *MockQueryCoord_UpdateStateCode_Call) RunAndReturn(run func(commonpb.St
return _c return _c
} }
// ValidateAnalyzer provides a mock function with given fields: _a0, _a1
func (_m *MockQueryCoord) ValidateAnalyzer(_a0 context.Context, _a1 *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error) {
ret := _m.Called(_a0, _a1)
if len(ret) == 0 {
panic("no return value specified for ValidateAnalyzer")
}
var r0 *commonpb.Status
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error)); ok {
return rf(_a0, _a1)
}
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest) *commonpb.Status); ok {
r0 = rf(_a0, _a1)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*commonpb.Status)
}
}
if rf, ok := ret.Get(1).(func(context.Context, *querypb.ValidateAnalyzerRequest) error); ok {
r1 = rf(_a0, _a1)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockQueryCoord_ValidateAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ValidateAnalyzer'
type MockQueryCoord_ValidateAnalyzer_Call struct {
*mock.Call
}
// ValidateAnalyzer is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *querypb.ValidateAnalyzerRequest
func (_e *MockQueryCoord_Expecter) ValidateAnalyzer(_a0 interface{}, _a1 interface{}) *MockQueryCoord_ValidateAnalyzer_Call {
return &MockQueryCoord_ValidateAnalyzer_Call{Call: _e.mock.On("ValidateAnalyzer", _a0, _a1)}
}
func (_c *MockQueryCoord_ValidateAnalyzer_Call) Run(run func(_a0 context.Context, _a1 *querypb.ValidateAnalyzerRequest)) *MockQueryCoord_ValidateAnalyzer_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(*querypb.ValidateAnalyzerRequest))
})
return _c
}
func (_c *MockQueryCoord_ValidateAnalyzer_Call) Return(_a0 *commonpb.Status, _a1 error) *MockQueryCoord_ValidateAnalyzer_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockQueryCoord_ValidateAnalyzer_Call) RunAndReturn(run func(context.Context, *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error)) *MockQueryCoord_ValidateAnalyzer_Call {
_c.Call.Return(run)
return _c
}
// NewMockQueryCoord creates a new instance of MockQueryCoord. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // NewMockQueryCoord creates a new instance of MockQueryCoord. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value. // The first argument is typically a *testing.T value.
func NewMockQueryCoord(t interface { func NewMockQueryCoord(t interface {

View File

@ -1926,6 +1926,80 @@ func (_c *MockQueryCoordClient_ResumeNode_Call) RunAndReturn(run func(context.Co
return _c return _c
} }
// RunAnalyzer provides a mock function with given fields: ctx, in, opts
func (_m *MockQueryCoordClient) RunAnalyzer(ctx context.Context, in *querypb.RunAnalyzerRequest, opts ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error) {
_va := make([]interface{}, len(opts))
for _i := range opts {
_va[_i] = opts[_i]
}
var _ca []interface{}
_ca = append(_ca, ctx, in)
_ca = append(_ca, _va...)
ret := _m.Called(_ca...)
if len(ret) == 0 {
panic("no return value specified for RunAnalyzer")
}
var r0 *milvuspb.RunAnalyzerResponse
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *querypb.RunAnalyzerRequest, ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error)); ok {
return rf(ctx, in, opts...)
}
if rf, ok := ret.Get(0).(func(context.Context, *querypb.RunAnalyzerRequest, ...grpc.CallOption) *milvuspb.RunAnalyzerResponse); ok {
r0 = rf(ctx, in, opts...)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*milvuspb.RunAnalyzerResponse)
}
}
if rf, ok := ret.Get(1).(func(context.Context, *querypb.RunAnalyzerRequest, ...grpc.CallOption) error); ok {
r1 = rf(ctx, in, opts...)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockQueryCoordClient_RunAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RunAnalyzer'
type MockQueryCoordClient_RunAnalyzer_Call struct {
*mock.Call
}
// RunAnalyzer is a helper method to define mock.On call
// - ctx context.Context
// - in *querypb.RunAnalyzerRequest
// - opts ...grpc.CallOption
func (_e *MockQueryCoordClient_Expecter) RunAnalyzer(ctx interface{}, in interface{}, opts ...interface{}) *MockQueryCoordClient_RunAnalyzer_Call {
return &MockQueryCoordClient_RunAnalyzer_Call{Call: _e.mock.On("RunAnalyzer",
append([]interface{}{ctx, in}, opts...)...)}
}
func (_c *MockQueryCoordClient_RunAnalyzer_Call) Run(run func(ctx context.Context, in *querypb.RunAnalyzerRequest, opts ...grpc.CallOption)) *MockQueryCoordClient_RunAnalyzer_Call {
_c.Call.Run(func(args mock.Arguments) {
variadicArgs := make([]grpc.CallOption, len(args)-2)
for i, a := range args[2:] {
if a != nil {
variadicArgs[i] = a.(grpc.CallOption)
}
}
run(args[0].(context.Context), args[1].(*querypb.RunAnalyzerRequest), variadicArgs...)
})
return _c
}
func (_c *MockQueryCoordClient_RunAnalyzer_Call) Return(_a0 *milvuspb.RunAnalyzerResponse, _a1 error) *MockQueryCoordClient_RunAnalyzer_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockQueryCoordClient_RunAnalyzer_Call) RunAndReturn(run func(context.Context, *querypb.RunAnalyzerRequest, ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error)) *MockQueryCoordClient_RunAnalyzer_Call {
_c.Call.Return(run)
return _c
}
// ShowConfigurations provides a mock function with given fields: ctx, in, opts // ShowConfigurations provides a mock function with given fields: ctx, in, opts
func (_m *MockQueryCoordClient) ShowConfigurations(ctx context.Context, in *internalpb.ShowConfigurationsRequest, opts ...grpc.CallOption) (*internalpb.ShowConfigurationsResponse, error) { func (_m *MockQueryCoordClient) ShowConfigurations(ctx context.Context, in *internalpb.ShowConfigurationsRequest, opts ...grpc.CallOption) (*internalpb.ShowConfigurationsResponse, error) {
_va := make([]interface{}, len(opts)) _va := make([]interface{}, len(opts))
@ -2814,6 +2888,80 @@ func (_c *MockQueryCoordClient_UpdateResourceGroups_Call) RunAndReturn(run func(
return _c return _c
} }
// ValidateAnalyzer provides a mock function with given fields: ctx, in, opts
func (_m *MockQueryCoordClient) ValidateAnalyzer(ctx context.Context, in *querypb.ValidateAnalyzerRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
_va := make([]interface{}, len(opts))
for _i := range opts {
_va[_i] = opts[_i]
}
var _ca []interface{}
_ca = append(_ca, ctx, in)
_ca = append(_ca, _va...)
ret := _m.Called(_ca...)
if len(ret) == 0 {
panic("no return value specified for ValidateAnalyzer")
}
var r0 *commonpb.Status
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest, ...grpc.CallOption) (*commonpb.Status, error)); ok {
return rf(ctx, in, opts...)
}
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest, ...grpc.CallOption) *commonpb.Status); ok {
r0 = rf(ctx, in, opts...)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*commonpb.Status)
}
}
if rf, ok := ret.Get(1).(func(context.Context, *querypb.ValidateAnalyzerRequest, ...grpc.CallOption) error); ok {
r1 = rf(ctx, in, opts...)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockQueryCoordClient_ValidateAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ValidateAnalyzer'
type MockQueryCoordClient_ValidateAnalyzer_Call struct {
*mock.Call
}
// ValidateAnalyzer is a helper method to define mock.On call
// - ctx context.Context
// - in *querypb.ValidateAnalyzerRequest
// - opts ...grpc.CallOption
func (_e *MockQueryCoordClient_Expecter) ValidateAnalyzer(ctx interface{}, in interface{}, opts ...interface{}) *MockQueryCoordClient_ValidateAnalyzer_Call {
return &MockQueryCoordClient_ValidateAnalyzer_Call{Call: _e.mock.On("ValidateAnalyzer",
append([]interface{}{ctx, in}, opts...)...)}
}
func (_c *MockQueryCoordClient_ValidateAnalyzer_Call) Run(run func(ctx context.Context, in *querypb.ValidateAnalyzerRequest, opts ...grpc.CallOption)) *MockQueryCoordClient_ValidateAnalyzer_Call {
_c.Call.Run(func(args mock.Arguments) {
variadicArgs := make([]grpc.CallOption, len(args)-2)
for i, a := range args[2:] {
if a != nil {
variadicArgs[i] = a.(grpc.CallOption)
}
}
run(args[0].(context.Context), args[1].(*querypb.ValidateAnalyzerRequest), variadicArgs...)
})
return _c
}
func (_c *MockQueryCoordClient_ValidateAnalyzer_Call) Return(_a0 *commonpb.Status, _a1 error) *MockQueryCoordClient_ValidateAnalyzer_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockQueryCoordClient_ValidateAnalyzer_Call) RunAndReturn(run func(context.Context, *querypb.ValidateAnalyzerRequest, ...grpc.CallOption) (*commonpb.Status, error)) *MockQueryCoordClient_ValidateAnalyzer_Call {
_c.Call.Return(run)
return _c
}
// NewMockQueryCoordClient creates a new instance of MockQueryCoordClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. // NewMockQueryCoordClient creates a new instance of MockQueryCoordClient. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
// The first argument is typically a *testing.T value. // The first argument is typically a *testing.T value.
func NewMockQueryCoordClient(t interface { func NewMockQueryCoordClient(t interface {

View File

@ -1968,6 +1968,65 @@ func (_c *MockQueryNode_UpdateStateCode_Call) RunAndReturn(run func(commonpb.Sta
return _c return _c
} }
// ValidateAnalyzer provides a mock function with given fields: _a0, _a1
func (_m *MockQueryNode) ValidateAnalyzer(_a0 context.Context, _a1 *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error) {
ret := _m.Called(_a0, _a1)
if len(ret) == 0 {
panic("no return value specified for ValidateAnalyzer")
}
var r0 *commonpb.Status
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error)); ok {
return rf(_a0, _a1)
}
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest) *commonpb.Status); ok {
r0 = rf(_a0, _a1)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*commonpb.Status)
}
}
if rf, ok := ret.Get(1).(func(context.Context, *querypb.ValidateAnalyzerRequest) error); ok {
r1 = rf(_a0, _a1)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockQueryNode_ValidateAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ValidateAnalyzer'
type MockQueryNode_ValidateAnalyzer_Call struct {
*mock.Call
}
// ValidateAnalyzer is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *querypb.ValidateAnalyzerRequest
func (_e *MockQueryNode_Expecter) ValidateAnalyzer(_a0 interface{}, _a1 interface{}) *MockQueryNode_ValidateAnalyzer_Call {
return &MockQueryNode_ValidateAnalyzer_Call{Call: _e.mock.On("ValidateAnalyzer", _a0, _a1)}
}
func (_c *MockQueryNode_ValidateAnalyzer_Call) Run(run func(_a0 context.Context, _a1 *querypb.ValidateAnalyzerRequest)) *MockQueryNode_ValidateAnalyzer_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(*querypb.ValidateAnalyzerRequest))
})
return _c
}
func (_c *MockQueryNode_ValidateAnalyzer_Call) Return(_a0 *commonpb.Status, _a1 error) *MockQueryNode_ValidateAnalyzer_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockQueryNode_ValidateAnalyzer_Call) RunAndReturn(run func(context.Context, *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error)) *MockQueryNode_ValidateAnalyzer_Call {
_c.Call.Return(run)
return _c
}
// WatchDmChannels provides a mock function with given fields: _a0, _a1 // WatchDmChannels provides a mock function with given fields: _a0, _a1
func (_m *MockQueryNode) WatchDmChannels(_a0 context.Context, _a1 *querypb.WatchDmChannelsRequest) (*commonpb.Status, error) { func (_m *MockQueryNode) WatchDmChannels(_a0 context.Context, _a1 *querypb.WatchDmChannelsRequest) (*commonpb.Status, error) {
ret := _m.Called(_a0, _a1) ret := _m.Called(_a0, _a1)

View File

@ -2074,6 +2074,80 @@ func (_c *MockQueryNodeClient_UpdateSchema_Call) RunAndReturn(run func(context.C
return _c return _c
} }
// ValidateAnalyzer provides a mock function with given fields: ctx, in, opts
func (_m *MockQueryNodeClient) ValidateAnalyzer(ctx context.Context, in *querypb.ValidateAnalyzerRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
_va := make([]interface{}, len(opts))
for _i := range opts {
_va[_i] = opts[_i]
}
var _ca []interface{}
_ca = append(_ca, ctx, in)
_ca = append(_ca, _va...)
ret := _m.Called(_ca...)
if len(ret) == 0 {
panic("no return value specified for ValidateAnalyzer")
}
var r0 *commonpb.Status
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest, ...grpc.CallOption) (*commonpb.Status, error)); ok {
return rf(ctx, in, opts...)
}
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest, ...grpc.CallOption) *commonpb.Status); ok {
r0 = rf(ctx, in, opts...)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*commonpb.Status)
}
}
if rf, ok := ret.Get(1).(func(context.Context, *querypb.ValidateAnalyzerRequest, ...grpc.CallOption) error); ok {
r1 = rf(ctx, in, opts...)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockQueryNodeClient_ValidateAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ValidateAnalyzer'
type MockQueryNodeClient_ValidateAnalyzer_Call struct {
*mock.Call
}
// ValidateAnalyzer is a helper method to define mock.On call
// - ctx context.Context
// - in *querypb.ValidateAnalyzerRequest
// - opts ...grpc.CallOption
func (_e *MockQueryNodeClient_Expecter) ValidateAnalyzer(ctx interface{}, in interface{}, opts ...interface{}) *MockQueryNodeClient_ValidateAnalyzer_Call {
return &MockQueryNodeClient_ValidateAnalyzer_Call{Call: _e.mock.On("ValidateAnalyzer",
append([]interface{}{ctx, in}, opts...)...)}
}
func (_c *MockQueryNodeClient_ValidateAnalyzer_Call) Run(run func(ctx context.Context, in *querypb.ValidateAnalyzerRequest, opts ...grpc.CallOption)) *MockQueryNodeClient_ValidateAnalyzer_Call {
_c.Call.Run(func(args mock.Arguments) {
variadicArgs := make([]grpc.CallOption, len(args)-2)
for i, a := range args[2:] {
if a != nil {
variadicArgs[i] = a.(grpc.CallOption)
}
}
run(args[0].(context.Context), args[1].(*querypb.ValidateAnalyzerRequest), variadicArgs...)
})
return _c
}
func (_c *MockQueryNodeClient_ValidateAnalyzer_Call) Return(_a0 *commonpb.Status, _a1 error) *MockQueryNodeClient_ValidateAnalyzer_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockQueryNodeClient_ValidateAnalyzer_Call) RunAndReturn(run func(context.Context, *querypb.ValidateAnalyzerRequest, ...grpc.CallOption) (*commonpb.Status, error)) *MockQueryNodeClient_ValidateAnalyzer_Call {
_c.Call.Return(run)
return _c
}
// WatchDmChannels provides a mock function with given fields: ctx, in, opts // WatchDmChannels provides a mock function with given fields: ctx, in, opts
func (_m *MockQueryNodeClient) WatchDmChannels(ctx context.Context, in *querypb.WatchDmChannelsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) { func (_m *MockQueryNodeClient) WatchDmChannels(ctx context.Context, in *querypb.WatchDmChannelsRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
_va := make([]interface{}, len(opts)) _va := make([]interface{}, len(opts))

View File

@ -46,7 +46,6 @@ import (
"github.com/milvus-io/milvus/internal/proxy/privilege" "github.com/milvus-io/milvus/internal/proxy/privilege"
"github.com/milvus-io/milvus/internal/proxy/replicate" "github.com/milvus-io/milvus/internal/proxy/replicate"
"github.com/milvus-io/milvus/internal/types" "github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/internal/util/analyzer"
"github.com/milvus-io/milvus/internal/util/hookutil" "github.com/milvus-io/milvus/internal/util/hookutil"
"github.com/milvus-io/milvus/internal/util/segcore" "github.com/milvus-io/milvus/internal/util/segcore"
"github.com/milvus-io/milvus/pkg/v2/common" "github.com/milvus-io/milvus/pkg/v2/common"
@ -6299,40 +6298,6 @@ func (node *Proxy) OperatePrivilegeGroup(ctx context.Context, req *milvuspb.Oper
return result, nil return result, nil
} }
func (node *Proxy) runAnalyzer(req *milvuspb.RunAnalyzerRequest) ([]*milvuspb.AnalyzerResult, error) {
analyzer, err := analyzer.NewAnalyzer(req.GetAnalyzerParams())
if err != nil {
return nil, err
}
defer analyzer.Destroy()
results := make([]*milvuspb.AnalyzerResult, len(req.GetPlaceholder()))
for i, text := range req.GetPlaceholder() {
stream := analyzer.NewTokenStream(string(text))
defer stream.Destroy()
results[i] = &milvuspb.AnalyzerResult{
Tokens: make([]*milvuspb.AnalyzerToken, 0),
}
for stream.Advance() {
var token *milvuspb.AnalyzerToken
if req.GetWithDetail() {
token = stream.DetailedToken()
} else {
token = &milvuspb.AnalyzerToken{Token: stream.Token()}
}
if req.GetWithHash() {
token.Hash = typeutil.HashString2LessUint32(token.GetToken())
}
results[i].Tokens = append(results[i].Tokens, token)
}
}
return results, nil
}
func (node *Proxy) RunAnalyzer(ctx context.Context, req *milvuspb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) { func (node *Proxy) RunAnalyzer(ctx context.Context, req *milvuspb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) {
ctx, sp := otel.Tracer(typeutil.ProxyRole).Start(ctx, "Proxy-RunAnalyzer") ctx, sp := otel.Tracer(typeutil.ProxyRole).Start(ctx, "Proxy-RunAnalyzer")
defer sp.End() defer sp.End()
@ -6350,20 +6315,19 @@ func (node *Proxy) RunAnalyzer(ctx context.Context, req *milvuspb.RunAnalyzerReq
}, nil }, nil
} }
// build and run analyzer at any streaming node/query node
// if collection and field not set
if req.GetCollectionName() == "" { if req.GetCollectionName() == "" {
results, err := node.runAnalyzer(req) return node.mixCoord.RunAnalyzer(ctx, &querypb.RunAnalyzerRequest{
if err != nil { AnalyzerParams: req.GetAnalyzerParams(),
return &milvuspb.RunAnalyzerResponse{ Placeholder: req.GetPlaceholder(),
Status: merr.Status(err), WithDetail: req.GetWithDetail(),
}, nil WithHash: req.GetWithHash(),
} })
return &milvuspb.RunAnalyzerResponse{
Status: merr.Status(nil),
Results: results,
}, nil
} }
// run builded analyzer by delegator
// collection must loaded
if err := validateRunAnalyzer(req); err != nil { if err := validateRunAnalyzer(req); err != nil {
return &milvuspb.RunAnalyzerResponse{ return &milvuspb.RunAnalyzerResponse{
Status: merr.Status(merr.WrapErrAsInputError(err)), Status: merr.Status(merr.WrapErrAsInputError(err)),

View File

@ -1664,19 +1664,26 @@ func TestRunAnalyzer(t *testing.T) {
}) })
p.UpdateStateCode(commonpb.StateCode_Healthy) p.UpdateStateCode(commonpb.StateCode_Healthy)
t.Run("run analyzer with default params", func(t *testing.T) { t.Run("run analyzer with mixcoord success", func(t *testing.T) {
mockMixcoord := mocks.NewMockMixCoordClient(t)
p.mixCoord = mockMixcoord
mockMixcoord.EXPECT().RunAnalyzer(mock.Anything, mock.Anything, mock.Anything).Return(&milvuspb.RunAnalyzerResponse{Status: merr.Status(nil)}, nil)
resp, err := p.RunAnalyzer(context.Background(), &milvuspb.RunAnalyzerRequest{ resp, err := p.RunAnalyzer(context.Background(), &milvuspb.RunAnalyzerRequest{
Placeholder: [][]byte{[]byte("test doc")}, Placeholder: [][]byte{[]byte("test doc")},
}) })
require.NoError(t, err) require.NoError(t, err)
require.NoError(t, merr.Error(resp.GetStatus())) require.NoError(t, merr.Error(resp.GetStatus()))
assert.Equal(t, len(resp.GetResults()[0].GetTokens()), 2)
}) })
t.Run("run analyzer with invalid params", func(t *testing.T) { t.Run("run analyzer with mixcoord failed", func(t *testing.T) {
mockMixcoord := mocks.NewMockMixCoordClient(t)
p.mixCoord = mockMixcoord
mockMixcoord.EXPECT().RunAnalyzer(mock.Anything, mock.Anything, mock.Anything).Return(&milvuspb.RunAnalyzerResponse{Status: merr.Status(fmt.Errorf("mock error"))}, nil)
resp, err := p.RunAnalyzer(context.Background(), &milvuspb.RunAnalyzerRequest{ resp, err := p.RunAnalyzer(context.Background(), &milvuspb.RunAnalyzerRequest{
Placeholder: [][]byte{[]byte("test doc")}, Placeholder: [][]byte{[]byte("test doc")},
AnalyzerParams: "invalid json",
}) })
require.NoError(t, err) require.NoError(t, err)
require.Error(t, merr.Error(resp.GetStatus())) require.Error(t, merr.Error(resp.GetStatus()))

View File

@ -1642,6 +1642,16 @@ func (coord *MixCoordMock) ListFileResources(ctx context.Context, req *milvuspb.
}, nil }, nil
} }
func (coord *MixCoordMock) RunAnalyzer(ctx context.Context, req *querypb.RunAnalyzerRequest, opts ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error) {
return &milvuspb.RunAnalyzerResponse{
Status: merr.Success(),
}, nil
}
func (coord *MixCoordMock) ValidateAnalyzer(ctx context.Context, req *querypb.ValidateAnalyzerRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
return merr.Success(), nil
}
type DescribeCollectionFunc func(ctx context.Context, request *milvuspb.DescribeCollectionRequest, opts ...grpc.CallOption) (*milvuspb.DescribeCollectionResponse, error) type DescribeCollectionFunc func(ctx context.Context, request *milvuspb.DescribeCollectionRequest, opts ...grpc.CallOption) (*milvuspb.DescribeCollectionResponse, error)
type ShowPartitionsFunc func(ctx context.Context, request *milvuspb.ShowPartitionsRequest, opts ...grpc.CallOption) (*milvuspb.ShowPartitionsResponse, error) type ShowPartitionsFunc func(ctx context.Context, request *milvuspb.ShowPartitionsRequest, opts ...grpc.CallOption) (*milvuspb.ShowPartitionsResponse, error)

View File

@ -325,7 +325,7 @@ func (lb *LBPolicyImpl) Execute(ctx context.Context, workload CollectionWorkLoad
return wg.Wait() return wg.Wait()
} }
// Execute will execute any one channel in collection workload // ExecuteOneChannel will execute at any one channel in collection
func (lb *LBPolicyImpl) ExecuteOneChannel(ctx context.Context, workload CollectionWorkLoad) error { func (lb *LBPolicyImpl) ExecuteOneChannel(ctx context.Context, workload CollectionWorkLoad) error {
channelList, err := lb.GetShardLeaderList(ctx, workload.Db, workload.CollectionName, workload.CollectionID, true) channelList, err := lb.GetShardLeaderList(ctx, workload.Db, workload.CollectionName, workload.CollectionID, true)
if err != nil { if err != nil {

View File

@ -40,7 +40,6 @@ import (
"github.com/milvus-io/milvus/internal/parser/planparserv2" "github.com/milvus-io/milvus/internal/parser/planparserv2"
"github.com/milvus-io/milvus/internal/proxy/privilege" "github.com/milvus-io/milvus/internal/proxy/privilege"
"github.com/milvus-io/milvus/internal/types" "github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/internal/util/analyzer"
"github.com/milvus-io/milvus/internal/util/function/embedding" "github.com/milvus-io/milvus/internal/util/function/embedding"
"github.com/milvus-io/milvus/internal/util/hookutil" "github.com/milvus-io/milvus/internal/util/hookutil"
"github.com/milvus-io/milvus/internal/util/indexparamcheck" "github.com/milvus-io/milvus/internal/util/indexparamcheck"
@ -632,10 +631,6 @@ func ValidateField(field *schemapb.FieldSchema, schema *schemapb.CollectionSchem
if err = ValidateAutoIndexMmapConfig(isVectorType, indexParams); err != nil { if err = ValidateAutoIndexMmapConfig(isVectorType, indexParams); err != nil {
return err return err
} }
if err := validateAnalyzer(schema, field); err != nil {
return err
}
return nil return nil
} }
@ -703,107 +698,6 @@ func ValidateStructArrayField(structArrayField *schemapb.StructArrayFieldSchema,
return nil return nil
} }
func validateMultiAnalyzerParams(params string, coll *schemapb.CollectionSchema) error {
var m map[string]json.RawMessage
var analyzerMap map[string]json.RawMessage
var mFileName string
err := json.Unmarshal([]byte(params), &m)
if err != nil {
return err
}
mfield, ok := m["by_field"]
if !ok {
return fmt.Errorf("multi analyzer params now must set by_field to specify with field decide analyzer")
}
err = json.Unmarshal(mfield, &mFileName)
if err != nil {
return fmt.Errorf("multi analyzer params by_field must be string but now: %s", mfield)
}
// check field exist
fieldExist := false
for _, field := range coll.GetFields() {
if field.GetName() == mFileName {
// only support string field now
if field.GetDataType() != schemapb.DataType_VarChar {
return fmt.Errorf("multi analyzer params now only support by string field, but field %s is not string", field.GetName())
}
fieldExist = true
break
}
}
if !fieldExist {
return fmt.Errorf("multi analyzer dependent field %s not exist in collection %s", string(mfield), coll.GetName())
}
if value, ok := m["alias"]; ok {
mapping := map[string]string{}
err = json.Unmarshal(value, &mapping)
if err != nil {
return fmt.Errorf("multi analyzer alias must be string map but now: %s", value)
}
}
analyzers, ok := m["analyzers"]
if !ok {
return fmt.Errorf("multi analyzer params must set analyzers ")
}
err = json.Unmarshal(analyzers, &analyzerMap)
if err != nil {
return fmt.Errorf("unmarshal analyzers failed: %s", err)
}
hasDefault := false
for name, params := range analyzerMap {
if err := analyzer.ValidateAnalyzer(string(params)); err != nil {
return fmt.Errorf("analyzer %s params invalid: %s", name, err)
}
if name == "default" {
hasDefault = true
}
}
if !hasDefault {
return fmt.Errorf("multi analyzer must set default analyzer for all unknown value")
}
return nil
}
func validateAnalyzer(collSchema *schemapb.CollectionSchema, fieldSchema *schemapb.FieldSchema) error {
h := typeutil.CreateFieldSchemaHelper(fieldSchema)
if !h.EnableMatch() && !wasBm25FunctionInputField(collSchema, fieldSchema) {
return nil
}
if !h.EnableAnalyzer() {
return fmt.Errorf("field %s is set to enable match or bm25 function but not enable analyzer", fieldSchema.Name)
}
if params, ok := h.GetMultiAnalyzerParams(); ok {
if h.EnableMatch() {
return fmt.Errorf("multi analyzer now only support for bm25, but now field %s enable match", fieldSchema.Name)
}
if h.HasAnalyzerParams() {
return fmt.Errorf("field %s analyzer params should be none if has multi analyzer params", fieldSchema.Name)
}
return validateMultiAnalyzerParams(params, collSchema)
}
for _, kv := range fieldSchema.GetTypeParams() {
if kv.GetKey() == "analyzer_params" {
return analyzer.ValidateAnalyzer(kv.Value)
}
}
// return nil when use default analyzer
return nil
}
func validatePrimaryKey(coll *schemapb.CollectionSchema) error { func validatePrimaryKey(coll *schemapb.CollectionSchema) error {
idx := -1 idx := -1
for i, field := range coll.Fields { for i, field := range coll.Fields {
@ -1222,7 +1116,7 @@ func fillFieldPropertiesBySchema(columns []*schemapb.FieldData, schema *schemapb
expectColumnNum := 0 expectColumnNum := 0
for _, field := range schema.GetFields() { for _, field := range schema.GetFields() {
fieldName2Schema[field.Name] = field fieldName2Schema[field.Name] = field
if !IsBM25FunctionOutputField(field, schema) { if !typeutil.IsBM25FunctionOutputField(field, schema) {
expectColumnNum++ expectColumnNum++
} }
} }
@ -1828,12 +1722,12 @@ func checkFieldsDataBySchema(allFields []*schemapb.FieldSchema, schema *schemapb
if fieldSchema.GetDefaultValue() != nil && fieldSchema.IsPrimaryKey { if fieldSchema.GetDefaultValue() != nil && fieldSchema.IsPrimaryKey {
return merr.WrapErrParameterInvalidMsg("primary key can't be with default value") return merr.WrapErrParameterInvalidMsg("primary key can't be with default value")
} }
if (fieldSchema.IsPrimaryKey && fieldSchema.AutoID && !Params.ProxyCfg.SkipAutoIDCheck.GetAsBool() && needAutoGenPk && inInsert) || IsBM25FunctionOutputField(fieldSchema, schema) { if (fieldSchema.IsPrimaryKey && fieldSchema.AutoID && !Params.ProxyCfg.SkipAutoIDCheck.GetAsBool() && needAutoGenPk && inInsert) || typeutil.IsBM25FunctionOutputField(fieldSchema, schema) {
// when inInsert, no need to pass when pk is autoid and SkipAutoIDCheck is false // when inInsert, no need to pass when pk is autoid and SkipAutoIDCheck is false
autoGenFieldNum++ autoGenFieldNum++
} }
if _, ok := dataNameSet[fieldSchema.GetName()]; !ok { if _, ok := dataNameSet[fieldSchema.GetName()]; !ok {
if (fieldSchema.IsPrimaryKey && fieldSchema.AutoID && !Params.ProxyCfg.SkipAutoIDCheck.GetAsBool() && needAutoGenPk && inInsert) || IsBM25FunctionOutputField(fieldSchema, schema) { if (fieldSchema.IsPrimaryKey && fieldSchema.AutoID && !Params.ProxyCfg.SkipAutoIDCheck.GetAsBool() && needAutoGenPk && inInsert) || typeutil.IsBM25FunctionOutputField(fieldSchema, schema) {
// autoGenField // autoGenField
continue continue
} }
@ -2044,7 +1938,7 @@ func LackOfFieldsDataBySchema(schema *schemapb.CollectionSchema, fieldsData []*s
if _, ok := dataNameMap[fieldSchema.GetName()]; !ok { if _, ok := dataNameMap[fieldSchema.GetName()]; !ok {
if (fieldSchema.IsPrimaryKey && fieldSchema.AutoID && !Params.ProxyCfg.SkipAutoIDCheck.GetAsBool() && skipPkFieldCheck) || if (fieldSchema.IsPrimaryKey && fieldSchema.AutoID && !Params.ProxyCfg.SkipAutoIDCheck.GetAsBool() && skipPkFieldCheck) ||
IsBM25FunctionOutputField(fieldSchema, schema) || typeutil.IsBM25FunctionOutputField(fieldSchema, schema) ||
(skipDynamicFieldCheck && fieldSchema.GetIsDynamic()) { (skipDynamicFieldCheck && fieldSchema.GetIsDynamic()) {
// autoGenField // autoGenField
continue continue
@ -2707,24 +2601,6 @@ func GetReplicateID(ctx context.Context, database, collectionName string) (strin
return replicateID, nil return replicateID, nil
} }
func IsBM25FunctionOutputField(field *schemapb.FieldSchema, collSchema *schemapb.CollectionSchema) bool {
if !(field.GetIsFunctionOutput() && field.GetDataType() == schemapb.DataType_SparseFloatVector) {
return false
}
for _, fSchema := range collSchema.Functions {
if fSchema.Type == schemapb.FunctionType_BM25 {
if len(fSchema.OutputFieldNames) != 0 && field.Name == fSchema.OutputFieldNames[0] {
return true
}
if len(fSchema.OutputFieldIds) != 0 && field.FieldID == fSchema.OutputFieldIds[0] {
return true
}
}
}
return false
}
func GetFunctionOutputFields(collSchema *schemapb.CollectionSchema) []string { func GetFunctionOutputFields(collSchema *schemapb.CollectionSchema) []string {
fields := make([]string, 0) fields := make([]string, 0)
for _, fSchema := range collSchema.Functions { for _, fSchema := range collSchema.Functions {

View File

@ -3086,25 +3086,6 @@ func TestValidateFunctionBasicParams(t *testing.T) {
}) })
} }
func TestIsBM25FunctionOutputField(t *testing.T) {
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{Name: "input_field", DataType: schemapb.DataType_VarChar, TypeParams: []*commonpb.KeyValuePair{{Key: "enable_analyzer", Value: "true"}}},
{Name: "output_field", DataType: schemapb.DataType_SparseFloatVector, IsFunctionOutput: true},
},
Functions: []*schemapb.FunctionSchema{
{
Name: "bm25_func",
Type: schemapb.FunctionType_BM25,
InputFieldNames: []string{"input_field"},
OutputFieldNames: []string{"output_field"},
},
},
}
assert.False(t, IsBM25FunctionOutputField(schema.Fields[0], schema))
assert.True(t, IsBM25FunctionOutputField(schema.Fields[1], schema))
}
func TestComputeRecall(t *testing.T) { func TestComputeRecall(t *testing.T) {
t.Run("normal case1", func(t *testing.T) { t.Run("normal case1", func(t *testing.T) {
result1 := &schemapb.SearchResultData{ result1 := &schemapb.SearchResultData{

View File

@ -1211,12 +1211,3 @@ func newValidateUtil(opts ...validateOption) *validateUtil {
func ValidateAutoIndexMmapConfig(isVectorField bool, indexParams map[string]string) error { func ValidateAutoIndexMmapConfig(isVectorField bool, indexParams map[string]string) error {
return common.ValidateAutoIndexMmapConfig(paramtable.Get().AutoIndexConfig.Enable.GetAsBool(), isVectorField, indexParams) return common.ValidateAutoIndexMmapConfig(paramtable.Get().AutoIndexConfig.Enable.GetAsBool(), isVectorField, indexParams)
} }
func wasBm25FunctionInputField(coll *schemapb.CollectionSchema, field *schemapb.FieldSchema) bool {
for _, fun := range coll.GetFunctions() {
if fun.GetType() == schemapb.FunctionType_BM25 && field.GetName() == fun.GetInputFieldNames()[0] {
return true
}
}
return false
}

View File

@ -1598,6 +1598,65 @@ func (_c *MockQueryNodeServer_UpdateSchema_Call) RunAndReturn(run func(context.C
return _c return _c
} }
// ValidateAnalyzer provides a mock function with given fields: _a0, _a1
func (_m *MockQueryNodeServer) ValidateAnalyzer(_a0 context.Context, _a1 *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error) {
ret := _m.Called(_a0, _a1)
if len(ret) == 0 {
panic("no return value specified for ValidateAnalyzer")
}
var r0 *commonpb.Status
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error)); ok {
return rf(_a0, _a1)
}
if rf, ok := ret.Get(0).(func(context.Context, *querypb.ValidateAnalyzerRequest) *commonpb.Status); ok {
r0 = rf(_a0, _a1)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*commonpb.Status)
}
}
if rf, ok := ret.Get(1).(func(context.Context, *querypb.ValidateAnalyzerRequest) error); ok {
r1 = rf(_a0, _a1)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockQueryNodeServer_ValidateAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ValidateAnalyzer'
type MockQueryNodeServer_ValidateAnalyzer_Call struct {
*mock.Call
}
// ValidateAnalyzer is a helper method to define mock.On call
// - _a0 context.Context
// - _a1 *querypb.ValidateAnalyzerRequest
func (_e *MockQueryNodeServer_Expecter) ValidateAnalyzer(_a0 interface{}, _a1 interface{}) *MockQueryNodeServer_ValidateAnalyzer_Call {
return &MockQueryNodeServer_ValidateAnalyzer_Call{Call: _e.mock.On("ValidateAnalyzer", _a0, _a1)}
}
func (_c *MockQueryNodeServer_ValidateAnalyzer_Call) Run(run func(_a0 context.Context, _a1 *querypb.ValidateAnalyzerRequest)) *MockQueryNodeServer_ValidateAnalyzer_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(*querypb.ValidateAnalyzerRequest))
})
return _c
}
func (_c *MockQueryNodeServer_ValidateAnalyzer_Call) Return(_a0 *commonpb.Status, _a1 error) *MockQueryNodeServer_ValidateAnalyzer_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockQueryNodeServer_ValidateAnalyzer_Call) RunAndReturn(run func(context.Context, *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error)) *MockQueryNodeServer_ValidateAnalyzer_Call {
_c.Call.Return(run)
return _c
}
// WatchDmChannels provides a mock function with given fields: _a0, _a1 // WatchDmChannels provides a mock function with given fields: _a0, _a1
func (_m *MockQueryNodeServer) WatchDmChannels(_a0 context.Context, _a1 *querypb.WatchDmChannelsRequest) (*commonpb.Status, error) { func (_m *MockQueryNodeServer) WatchDmChannels(_a0 context.Context, _a1 *querypb.WatchDmChannelsRequest) (*commonpb.Status, error) {
ret := _m.Called(_a0, _a1) ret := _m.Called(_a0, _a1)

View File

@ -129,6 +129,10 @@ type Server struct {
proxyClientManager proxyutil.ProxyClientManagerInterface proxyClientManager proxyutil.ProxyClientManagerInterface
metricsRequest *metricsinfo.MetricsRequest metricsRequest *metricsinfo.MetricsRequest
// for balance streaming node request
// now only used for run analyzer and validate analyzer
nodeIdx atomic.Uint32
} }
func NewQueryCoord(ctx context.Context) (*Server, error) { func NewQueryCoord(ctx context.Context) (*Server, error) {

View File

@ -29,6 +29,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/internal/coordinator/snmanager"
"github.com/milvus-io/milvus/internal/querycoordv2/job" "github.com/milvus-io/milvus/internal/querycoordv2/job"
"github.com/milvus-io/milvus/internal/querycoordv2/meta" "github.com/milvus-io/milvus/internal/querycoordv2/meta"
"github.com/milvus-io/milvus/internal/querycoordv2/utils" "github.com/milvus-io/milvus/internal/querycoordv2/utils"
@ -1289,3 +1290,47 @@ func (s *Server) ListLoadedSegments(ctx context.Context, req *querypb.ListLoaded
} }
return resp, nil return resp, nil
} }
func (s *Server) RunAnalyzer(ctx context.Context, req *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) {
if err := merr.CheckHealthy(s.State()); err != nil {
return &milvuspb.RunAnalyzerResponse{
Status: merr.Status(errors.Wrap(err, "failed to run analyzer")),
}, nil
}
nodeIDs := snmanager.StaticStreamingNodeManager.GetStreamingQueryNodeIDs().Collect()
if len(nodeIDs) == 0 {
return &milvuspb.RunAnalyzerResponse{
Status: merr.Status(errors.New("failed to validate analyzer, no delegator")),
}, nil
}
idx := s.nodeIdx.Inc() % uint32(len(nodeIDs))
resp, err := s.cluster.RunAnalyzer(ctx, nodeIDs[idx], req)
if err != nil {
return &milvuspb.RunAnalyzerResponse{
Status: merr.Status(err),
}, nil
}
return resp, nil
}
func (s *Server) ValidateAnalyzer(ctx context.Context, req *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error) {
if err := merr.CheckHealthy(s.State()); err != nil {
return merr.Status(errors.Wrap(err, "failed to validate analyzer")), nil
}
nodeIDs := snmanager.StaticStreamingNodeManager.GetStreamingQueryNodeIDs().Collect()
if len(nodeIDs) == 0 {
return merr.Status(errors.New("failed to validate analyzer, no delegator")), nil
}
idx := s.nodeIdx.Inc() % uint32(len(nodeIDs))
resp, err := s.cluster.ValidateAnalyzer(ctx, nodeIDs[idx], req)
if err != nil {
return merr.Status(err), nil
}
return resp, nil
}

View File

@ -53,6 +53,8 @@ type Cluster interface {
SyncDistribution(ctx context.Context, nodeID int64, req *querypb.SyncDistributionRequest) (*commonpb.Status, error) SyncDistribution(ctx context.Context, nodeID int64, req *querypb.SyncDistributionRequest) (*commonpb.Status, error)
GetComponentStates(ctx context.Context, nodeID int64) (*milvuspb.ComponentStates, error) GetComponentStates(ctx context.Context, nodeID int64) (*milvuspb.ComponentStates, error)
DropIndex(ctx context.Context, nodeID int64, req *querypb.DropIndexRequest) (*commonpb.Status, error) DropIndex(ctx context.Context, nodeID int64, req *querypb.DropIndexRequest) (*commonpb.Status, error)
RunAnalyzer(ctx context.Context, nodeID int64, req *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error)
ValidateAnalyzer(ctx context.Context, nodeID int64, req *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error)
Start() Start()
Stop() Stop()
} }
@ -283,6 +285,36 @@ func (c *QueryCluster) DropIndex(ctx context.Context, nodeID int64, req *querypb
return resp, err return resp, err
} }
func (c *QueryCluster) RunAnalyzer(ctx context.Context, nodeID int64, req *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) {
var (
resp *milvuspb.RunAnalyzerResponse
err error
)
sendErr := c.send(ctx, nodeID, func(cli types.QueryNodeClient) {
resp, err = cli.RunAnalyzer(ctx, req)
})
if sendErr != nil {
return nil, sendErr
}
return resp, err
}
func (c *QueryCluster) ValidateAnalyzer(ctx context.Context, nodeID int64, req *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error) {
var (
resp *commonpb.Status
err error
)
sendErr := c.send(ctx, nodeID, func(cli types.QueryNodeClient) {
resp, err = cli.ValidateAnalyzer(ctx, req)
})
if sendErr != nil {
return nil, sendErr
}
return resp, err
}
func (c *QueryCluster) send(ctx context.Context, nodeID int64, fn func(cli types.QueryNodeClient)) error { func (c *QueryCluster) send(ctx context.Context, nodeID int64, fn func(cli types.QueryNodeClient)) error {
node := c.nodeManager.Get(nodeID) node := c.nodeManager.Get(nodeID)
if node == nil { if node == nil {

View File

@ -506,6 +506,66 @@ func (_c *MockCluster_ReleaseSegments_Call) RunAndReturn(run func(context.Contex
return _c return _c
} }
// RunAnalyzer provides a mock function with given fields: ctx, nodeID, req
func (_m *MockCluster) RunAnalyzer(ctx context.Context, nodeID int64, req *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) {
ret := _m.Called(ctx, nodeID, req)
if len(ret) == 0 {
panic("no return value specified for RunAnalyzer")
}
var r0 *milvuspb.RunAnalyzerResponse
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, int64, *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error)); ok {
return rf(ctx, nodeID, req)
}
if rf, ok := ret.Get(0).(func(context.Context, int64, *querypb.RunAnalyzerRequest) *milvuspb.RunAnalyzerResponse); ok {
r0 = rf(ctx, nodeID, req)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*milvuspb.RunAnalyzerResponse)
}
}
if rf, ok := ret.Get(1).(func(context.Context, int64, *querypb.RunAnalyzerRequest) error); ok {
r1 = rf(ctx, nodeID, req)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockCluster_RunAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RunAnalyzer'
type MockCluster_RunAnalyzer_Call struct {
*mock.Call
}
// RunAnalyzer is a helper method to define mock.On call
// - ctx context.Context
// - nodeID int64
// - req *querypb.RunAnalyzerRequest
func (_e *MockCluster_Expecter) RunAnalyzer(ctx interface{}, nodeID interface{}, req interface{}) *MockCluster_RunAnalyzer_Call {
return &MockCluster_RunAnalyzer_Call{Call: _e.mock.On("RunAnalyzer", ctx, nodeID, req)}
}
func (_c *MockCluster_RunAnalyzer_Call) Run(run func(ctx context.Context, nodeID int64, req *querypb.RunAnalyzerRequest)) *MockCluster_RunAnalyzer_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(int64), args[2].(*querypb.RunAnalyzerRequest))
})
return _c
}
func (_c *MockCluster_RunAnalyzer_Call) Return(_a0 *milvuspb.RunAnalyzerResponse, _a1 error) *MockCluster_RunAnalyzer_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockCluster_RunAnalyzer_Call) RunAndReturn(run func(context.Context, int64, *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error)) *MockCluster_RunAnalyzer_Call {
_c.Call.Return(run)
return _c
}
// Start provides a mock function with no fields // Start provides a mock function with no fields
func (_m *MockCluster) Start() { func (_m *MockCluster) Start() {
_m.Called() _m.Called()
@ -690,6 +750,66 @@ func (_c *MockCluster_UnsubDmChannel_Call) RunAndReturn(run func(context.Context
return _c return _c
} }
// ValidateAnalyzer provides a mock function with given fields: ctx, nodeID, req
func (_m *MockCluster) ValidateAnalyzer(ctx context.Context, nodeID int64, req *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error) {
ret := _m.Called(ctx, nodeID, req)
if len(ret) == 0 {
panic("no return value specified for ValidateAnalyzer")
}
var r0 *commonpb.Status
var r1 error
if rf, ok := ret.Get(0).(func(context.Context, int64, *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error)); ok {
return rf(ctx, nodeID, req)
}
if rf, ok := ret.Get(0).(func(context.Context, int64, *querypb.ValidateAnalyzerRequest) *commonpb.Status); ok {
r0 = rf(ctx, nodeID, req)
} else {
if ret.Get(0) != nil {
r0 = ret.Get(0).(*commonpb.Status)
}
}
if rf, ok := ret.Get(1).(func(context.Context, int64, *querypb.ValidateAnalyzerRequest) error); ok {
r1 = rf(ctx, nodeID, req)
} else {
r1 = ret.Error(1)
}
return r0, r1
}
// MockCluster_ValidateAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ValidateAnalyzer'
type MockCluster_ValidateAnalyzer_Call struct {
*mock.Call
}
// ValidateAnalyzer is a helper method to define mock.On call
// - ctx context.Context
// - nodeID int64
// - req *querypb.ValidateAnalyzerRequest
func (_e *MockCluster_Expecter) ValidateAnalyzer(ctx interface{}, nodeID interface{}, req interface{}) *MockCluster_ValidateAnalyzer_Call {
return &MockCluster_ValidateAnalyzer_Call{Call: _e.mock.On("ValidateAnalyzer", ctx, nodeID, req)}
}
func (_c *MockCluster_ValidateAnalyzer_Call) Run(run func(ctx context.Context, nodeID int64, req *querypb.ValidateAnalyzerRequest)) *MockCluster_ValidateAnalyzer_Call {
_c.Call.Run(func(args mock.Arguments) {
run(args[0].(context.Context), args[1].(int64), args[2].(*querypb.ValidateAnalyzerRequest))
})
return _c
}
func (_c *MockCluster_ValidateAnalyzer_Call) Return(_a0 *commonpb.Status, _a1 error) *MockCluster_ValidateAnalyzer_Call {
_c.Call.Return(_a0, _a1)
return _c
}
func (_c *MockCluster_ValidateAnalyzer_Call) RunAndReturn(run func(context.Context, int64, *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error)) *MockCluster_ValidateAnalyzer_Call {
_c.Call.Return(run)
return _c
}
// WatchDmChannels provides a mock function with given fields: ctx, nodeID, req // WatchDmChannels provides a mock function with given fields: ctx, nodeID, req
func (_m *MockCluster) WatchDmChannels(ctx context.Context, nodeID int64, req *querypb.WatchDmChannelsRequest) (*commonpb.Status, error) { func (_m *MockCluster) WatchDmChannels(ctx context.Context, nodeID int64, req *querypb.WatchDmChannelsRequest) (*commonpb.Status, error) {
ret := _m.Called(ctx, nodeID, req) ret := _m.Called(ctx, nodeID, req)

View File

@ -37,6 +37,7 @@ import (
"github.com/milvus-io/milvus/internal/querynodev2/segments" "github.com/milvus-io/milvus/internal/querynodev2/segments"
"github.com/milvus-io/milvus/internal/querynodev2/tasks" "github.com/milvus-io/milvus/internal/querynodev2/tasks"
"github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/analyzer"
"github.com/milvus-io/milvus/internal/util/searchutil/scheduler" "github.com/milvus-io/milvus/internal/util/searchutil/scheduler"
"github.com/milvus-io/milvus/internal/util/streamrpc" "github.com/milvus-io/milvus/internal/util/streamrpc"
"github.com/milvus-io/milvus/pkg/v2/common" "github.com/milvus-io/milvus/pkg/v2/common"
@ -1572,7 +1573,60 @@ func (node *QueryNode) DeleteBatch(ctx context.Context, req *querypb.DeleteBatch
}, nil }, nil
} }
func (node *QueryNode) runAnalyzer(req *querypb.RunAnalyzerRequest) ([]*milvuspb.AnalyzerResult, error) {
tokenizer, err := analyzer.NewAnalyzer(req.GetAnalyzerParams())
if err != nil {
return nil, err
}
defer tokenizer.Destroy()
results := make([]*milvuspb.AnalyzerResult, len(req.GetPlaceholder()))
for i, text := range req.GetPlaceholder() {
stream := tokenizer.NewTokenStream(string(text))
defer stream.Destroy()
results[i] = &milvuspb.AnalyzerResult{
Tokens: make([]*milvuspb.AnalyzerToken, 0),
}
for stream.Advance() {
var token *milvuspb.AnalyzerToken
if req.GetWithDetail() {
token = stream.DetailedToken()
} else {
token = &milvuspb.AnalyzerToken{Token: stream.Token()}
}
if req.GetWithHash() {
token.Hash = typeutil.HashString2LessUint32(token.GetToken())
}
results[i].Tokens = append(results[i].Tokens, token)
}
}
return results, nil
}
func (node *QueryNode) RunAnalyzer(ctx context.Context, req *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) { func (node *QueryNode) RunAnalyzer(ctx context.Context, req *querypb.RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) {
// check node healthy
if err := node.lifetime.Add(merr.IsHealthy); err != nil {
return &milvuspb.RunAnalyzerResponse{
Status: merr.Status(err),
}, nil
}
defer node.lifetime.Done()
// build and run analyzer by analyzer params
// if channel not set
if req.GetChannel() == "" {
result, err := node.runAnalyzer(req)
return &milvuspb.RunAnalyzerResponse{
Status: merr.Status(err),
Results: result,
}, nil
}
// run analyzer by delegator
// get delegator // get delegator
sd, ok := node.delegators.Get(req.GetChannel()) sd, ok := node.delegators.Get(req.GetChannel())
if !ok { if !ok {
@ -1597,6 +1651,26 @@ func (node *QueryNode) RunAnalyzer(ctx context.Context, req *querypb.RunAnalyzer
}, nil }, nil
} }
func (node *QueryNode) ValidateAnalyzer(ctx context.Context, req *querypb.ValidateAnalyzerRequest) (*commonpb.Status, error) {
// check node healthy
if err := node.lifetime.Add(merr.IsHealthy); err != nil {
return merr.Status(err), nil
}
defer node.lifetime.Done()
for _, info := range req.AnalyzerInfos {
err := analyzer.ValidateAnalyzer(info.GetParams())
if err != nil {
if info.GetName() != "" {
return merr.Status(merr.WrapErrParameterInvalidMsg("validate analyzer failed for field: %s, name: %s, error: %v", info.GetField(), info.GetName(), err)), nil
}
return merr.Status(merr.WrapErrParameterInvalidMsg("validate analyzer failed for field: %s, error: %v", info.GetField(), err)), nil
}
}
return merr.Status(nil), nil
}
type deleteRequestStringer struct { type deleteRequestStringer struct {
*querypb.DeleteRequest *querypb.DeleteRequest
} }

View File

@ -2415,6 +2415,60 @@ func (suite *ServiceSuite) TestRunAnalyzer() {
}) })
} }
func (suite *ServiceSuite) TestValidateAnalyzer() {
ctx := context.Background()
suite.Run("normal validate", func() {
req := &querypb.ValidateAnalyzerRequest{
AnalyzerInfos: []*querypb.AnalyzerInfo{
{
Field: "test_field",
Name: "test_analyzer",
Params: `{}`,
},
},
}
resp, err := suite.node.ValidateAnalyzer(ctx, req)
suite.Require().NoError(err)
suite.Require().Equal(commonpb.ErrorCode_Success, resp.GetErrorCode())
})
suite.Run("invalid analyzer params", func() {
req := &querypb.ValidateAnalyzerRequest{
AnalyzerInfos: []*querypb.AnalyzerInfo{
{
Field: "test_field",
Name: "test_analyzer",
Params: `{"invalid": "params"}`,
},
},
}
resp, err := suite.node.ValidateAnalyzer(ctx, req)
suite.Require().NoError(err)
suite.Require().NotEqual(commonpb.ErrorCode_Success, resp.GetErrorCode())
})
suite.Run("abnormal node", func() {
suite.node.UpdateStateCode(commonpb.StateCode_Abnormal)
defer suite.node.UpdateStateCode(commonpb.StateCode_Healthy)
req := &querypb.ValidateAnalyzerRequest{
AnalyzerInfos: []*querypb.AnalyzerInfo{
{
Field: "test_field",
Params: `{}`,
},
},
}
resp, err := suite.node.ValidateAnalyzer(ctx, req)
suite.Require().NoError(err)
suite.Require().NotEqual(commonpb.ErrorCode_Success, resp.GetErrorCode())
})
}
func TestQueryNodeService(t *testing.T) { func TestQueryNodeService(t *testing.T) {
wal := mock_streaming.NewMockWALAccesser(t) wal := mock_streaming.NewMockWALAccesser(t)
local := mock_streaming.NewMockLocal(t) local := mock_streaming.NewMockLocal(t)

View File

@ -33,6 +33,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/internal/coordinator/snmanager" "github.com/milvus-io/milvus/internal/coordinator/snmanager"
"github.com/milvus-io/milvus/internal/distributed/streaming" "github.com/milvus-io/milvus/internal/distributed/streaming"
"github.com/milvus-io/milvus/internal/json"
"github.com/milvus-io/milvus/internal/metastore/model" "github.com/milvus-io/milvus/internal/metastore/model"
"github.com/milvus-io/milvus/internal/util/hookutil" "github.com/milvus-io/milvus/internal/util/hookutil"
"github.com/milvus-io/milvus/internal/util/proxyutil" "github.com/milvus-io/milvus/internal/util/proxyutil"
@ -41,6 +42,7 @@ import (
"github.com/milvus-io/milvus/pkg/v2/log" "github.com/milvus-io/milvus/pkg/v2/log"
ms "github.com/milvus-io/milvus/pkg/v2/mq/msgstream" ms "github.com/milvus-io/milvus/pkg/v2/mq/msgstream"
pb "github.com/milvus-io/milvus/pkg/v2/proto/etcdpb" pb "github.com/milvus-io/milvus/pkg/v2/proto/etcdpb"
"github.com/milvus-io/milvus/pkg/v2/proto/querypb"
"github.com/milvus-io/milvus/pkg/v2/streaming/util/message" "github.com/milvus-io/milvus/pkg/v2/streaming/util/message"
"github.com/milvus-io/milvus/pkg/v2/streaming/util/message/adaptor" "github.com/milvus-io/milvus/pkg/v2/streaming/util/message/adaptor"
"github.com/milvus-io/milvus/pkg/v2/util/commonpbutil" "github.com/milvus-io/milvus/pkg/v2/util/commonpbutil"
@ -116,6 +118,7 @@ func (t *createCollectionTask) validate(ctx context.Context) error {
if t.Req.GetNumPartitions() > 0 { if t.Req.GetNumPartitions() > 0 {
newPartNum = t.Req.GetNumPartitions() newPartNum = t.Req.GetNumPartitions()
} }
return checkGeneralCapacity(t.ctx, 1, newPartNum, t.Req.GetShardsNum(), t.core) return checkGeneralCapacity(t.ctx, 1, newPartNum, t.Req.GetShardsNum(), t.core)
} }
@ -210,6 +213,29 @@ func (t *createCollectionTask) validateSchema(ctx context.Context, schema *schem
return err return err
} }
// check analyzer was vaild
analyzerInfos := make([]*querypb.AnalyzerInfo, 0)
for _, field := range schema.GetFields() {
err := validateAnalyzer(schema, field, &analyzerInfos)
if err != nil {
return err
}
}
// validate analyzer params at any streaming node
if len(analyzerInfos) > 0 {
resp, err := t.core.mixCoord.ValidateAnalyzer(t.ctx, &querypb.ValidateAnalyzerRequest{
AnalyzerInfos: analyzerInfos,
})
if err != nil {
return err
}
if err := merr.Error(resp); err != nil {
return err
}
}
return validateFieldDataType(schema.GetFields()) return validateFieldDataType(schema.GetFields())
} }
@ -768,3 +794,109 @@ func executeCreateCollectionTaskSteps(ctx context.Context,
}, &nullStep{}) // We'll remove the whole collection anyway. }, &nullStep{}) // We'll remove the whole collection anyway.
return undoTask.Execute(ctx) return undoTask.Execute(ctx)
} }
func validateMultiAnalyzerParams(params string, coll *schemapb.CollectionSchema, fieldSchema *schemapb.FieldSchema, infos *[]*querypb.AnalyzerInfo) error {
var m map[string]json.RawMessage
var analyzerMap map[string]json.RawMessage
var mFileName string
err := json.Unmarshal([]byte(params), &m)
if err != nil {
return err
}
mfield, ok := m["by_field"]
if !ok {
return fmt.Errorf("multi analyzer params now must set by_field to specify with field decide analyzer")
}
err = json.Unmarshal(mfield, &mFileName)
if err != nil {
return fmt.Errorf("multi analyzer params by_field must be string but now: %s", mfield)
}
// check field exist
fieldExist := false
for _, field := range coll.GetFields() {
if field.GetName() == mFileName {
// only support string field now
if field.GetDataType() != schemapb.DataType_VarChar {
return fmt.Errorf("multi analyzer params now only support by string field, but field %s is not string", field.GetName())
}
fieldExist = true
break
}
}
if !fieldExist {
return fmt.Errorf("multi analyzer dependent field %s not exist in collection %s", string(mfield), coll.GetName())
}
if value, ok := m["alias"]; ok {
mapping := map[string]string{}
err = json.Unmarshal(value, &mapping)
if err != nil {
return fmt.Errorf("multi analyzer alias must be string map but now: %s", value)
}
}
analyzers, ok := m["analyzers"]
if !ok {
return fmt.Errorf("multi analyzer params must set analyzers ")
}
err = json.Unmarshal(analyzers, &analyzerMap)
if err != nil {
return fmt.Errorf("unmarshal analyzers failed: %s", err)
}
hasDefault := false
for name, bytes := range analyzerMap {
*infos = append(*infos, &querypb.AnalyzerInfo{
Name: name,
Field: fieldSchema.GetName(),
Params: string(bytes),
})
if name == "default" {
hasDefault = true
}
}
if !hasDefault {
return fmt.Errorf("multi analyzer must set default analyzer for all unknown value")
}
return nil
}
func validateAnalyzer(collSchema *schemapb.CollectionSchema, fieldSchema *schemapb.FieldSchema, analyzerInfos *[]*querypb.AnalyzerInfo) error {
h := typeutil.CreateFieldSchemaHelper(fieldSchema)
if !h.EnableMatch() && !typeutil.IsBm25FunctionInputField(collSchema, fieldSchema) {
return nil
}
if !h.EnableAnalyzer() {
return fmt.Errorf("field %s which has enable_match or is input of BM25 function must also enable_analyzer", fieldSchema.Name)
}
if params, ok := h.GetMultiAnalyzerParams(); ok {
if h.EnableMatch() {
return fmt.Errorf("multi analyzer now only support for bm25, but now field %s enable match", fieldSchema.Name)
}
if h.HasAnalyzerParams() {
return fmt.Errorf("field %s analyzer params should be none if has multi analyzer params", fieldSchema.Name)
}
return validateMultiAnalyzerParams(params, collSchema, fieldSchema, analyzerInfos)
}
for _, kv := range fieldSchema.GetTypeParams() {
if kv.GetKey() == "analyzer_params" {
*analyzerInfos = append(*analyzerInfos, &querypb.AnalyzerInfo{
Field: fieldSchema.GetName(),
Params: kv.GetValue(),
})
}
}
// return nil when use default analyzer
return nil
}

View File

@ -27,6 +27,7 @@ import (
"github.com/cockroachdb/errors" "github.com/cockroachdb/errors"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock" "github.com/stretchr/testify/mock"
"github.com/stretchr/testify/require"
"google.golang.org/protobuf/proto" "google.golang.org/protobuf/proto"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb" "github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
@ -37,6 +38,7 @@ import (
mockrootcoord "github.com/milvus-io/milvus/internal/rootcoord/mocks" mockrootcoord "github.com/milvus-io/milvus/internal/rootcoord/mocks"
"github.com/milvus-io/milvus/pkg/v2/common" "github.com/milvus-io/milvus/pkg/v2/common"
"github.com/milvus-io/milvus/pkg/v2/proto/datapb" "github.com/milvus-io/milvus/pkg/v2/proto/datapb"
"github.com/milvus-io/milvus/pkg/v2/proto/querypb"
"github.com/milvus-io/milvus/pkg/v2/util" "github.com/milvus-io/milvus/pkg/v2/util"
"github.com/milvus-io/milvus/pkg/v2/util/funcutil" "github.com/milvus-io/milvus/pkg/v2/util/funcutil"
"github.com/milvus-io/milvus/pkg/v2/util/merr" "github.com/milvus-io/milvus/pkg/v2/util/merr"
@ -1800,3 +1802,355 @@ func TestNamespaceProperty(t *testing.T) {
assert.Error(t, err) assert.Error(t, err)
}) })
} }
func Test_validateMultiAnalyzerParams(t *testing.T) {
createTestCollectionSchema := func(fields []*schemapb.FieldSchema) *schemapb.CollectionSchema {
return &schemapb.CollectionSchema{
Name: "test_collection",
Fields: fields,
}
}
createTestFieldSchema := func(name string, dataType schemapb.DataType) *schemapb.FieldSchema {
return &schemapb.FieldSchema{
Name: name,
DataType: dataType,
}
}
t.Run("invalid json params", func(t *testing.T) {
coll := createTestCollectionSchema([]*schemapb.FieldSchema{})
fieldSchema := createTestFieldSchema("test_field", schemapb.DataType_VarChar)
infos := make([]*querypb.AnalyzerInfo, 0)
err := validateMultiAnalyzerParams("invalid json", coll, fieldSchema, &infos)
assert.Error(t, err)
})
t.Run("missing by_field", func(t *testing.T) {
coll := createTestCollectionSchema([]*schemapb.FieldSchema{})
fieldSchema := createTestFieldSchema("test_field", schemapb.DataType_VarChar)
infos := make([]*querypb.AnalyzerInfo, 0)
params := `{"analyzers": {"default": {}}}`
err := validateMultiAnalyzerParams(params, coll, fieldSchema, &infos)
assert.Error(t, err)
})
t.Run("by_field not string", func(t *testing.T) {
coll := createTestCollectionSchema([]*schemapb.FieldSchema{})
fieldSchema := createTestFieldSchema("test_field", schemapb.DataType_VarChar)
infos := make([]*querypb.AnalyzerInfo, 0)
params := `{"by_field": 123, "analyzers": {"default": {}}}`
err := validateMultiAnalyzerParams(params, coll, fieldSchema, &infos)
assert.Error(t, err)
})
t.Run("by_field references non-existent field", func(t *testing.T) {
coll := createTestCollectionSchema([]*schemapb.FieldSchema{
createTestFieldSchema("existing_field", schemapb.DataType_VarChar),
})
fieldSchema := createTestFieldSchema("test_field", schemapb.DataType_VarChar)
infos := make([]*querypb.AnalyzerInfo, 0)
params := `{"by_field": "non_existent_field", "analyzers": {"default": {}}}`
err := validateMultiAnalyzerParams(params, coll, fieldSchema, &infos)
assert.Error(t, err)
})
t.Run("by_field references non-string field", func(t *testing.T) {
coll := createTestCollectionSchema([]*schemapb.FieldSchema{
createTestFieldSchema("int_field", schemapb.DataType_Int64),
})
fieldSchema := createTestFieldSchema("test_field", schemapb.DataType_VarChar)
infos := make([]*querypb.AnalyzerInfo, 0)
params := `{"by_field": "int_field", "analyzers": {"default": {}}}`
err := validateMultiAnalyzerParams(params, coll, fieldSchema, &infos)
assert.Error(t, err)
})
t.Run("invalid alias format", func(t *testing.T) {
coll := createTestCollectionSchema([]*schemapb.FieldSchema{
createTestFieldSchema("string_field", schemapb.DataType_VarChar),
})
fieldSchema := createTestFieldSchema("test_field", schemapb.DataType_VarChar)
infos := make([]*querypb.AnalyzerInfo, 0)
params := `{"by_field": "string_field", "alias": "invalid_alias", "analyzers": {"default": {}}}`
err := validateMultiAnalyzerParams(params, coll, fieldSchema, &infos)
assert.Error(t, err)
})
t.Run("missing analyzers", func(t *testing.T) {
coll := createTestCollectionSchema([]*schemapb.FieldSchema{
createTestFieldSchema("string_field", schemapb.DataType_VarChar),
})
fieldSchema := createTestFieldSchema("test_field", schemapb.DataType_VarChar)
infos := make([]*querypb.AnalyzerInfo, 0)
params := `{"by_field": "string_field"}`
err := validateMultiAnalyzerParams(params, coll, fieldSchema, &infos)
assert.Error(t, err)
})
t.Run("invalid analyzers format", func(t *testing.T) {
coll := createTestCollectionSchema([]*schemapb.FieldSchema{
createTestFieldSchema("string_field", schemapb.DataType_VarChar),
})
fieldSchema := createTestFieldSchema("test_field", schemapb.DataType_VarChar)
infos := make([]*querypb.AnalyzerInfo, 0)
params := `{"by_field": "string_field", "analyzers": "invalid_analyzers"}`
err := validateMultiAnalyzerParams(params, coll, fieldSchema, &infos)
assert.Error(t, err)
})
t.Run("missing default analyzer", func(t *testing.T) {
coll := createTestCollectionSchema([]*schemapb.FieldSchema{
createTestFieldSchema("string_field", schemapb.DataType_VarChar),
})
fieldSchema := createTestFieldSchema("test_field", schemapb.DataType_VarChar)
infos := make([]*querypb.AnalyzerInfo, 0)
params := `{"by_field": "string_field", "analyzers": {"custom": {}}}`
err := validateMultiAnalyzerParams(params, coll, fieldSchema, &infos)
assert.Error(t, err)
})
t.Run("valid params", func(t *testing.T) {
coll := createTestCollectionSchema([]*schemapb.FieldSchema{
createTestFieldSchema("string_field", schemapb.DataType_VarChar),
})
fieldSchema := createTestFieldSchema("test_field", schemapb.DataType_VarChar)
infos := make([]*querypb.AnalyzerInfo, 0)
params := `{
"by_field": "string_field",
"alias": {"en": "english", "zh": "chinese"},
"analyzers": {
"default": {"type": "standard"},
"english": {"type": "english"},
"chinese": {"type": "chinese"}
}
}`
err := validateMultiAnalyzerParams(params, coll, fieldSchema, &infos)
assert.NoError(t, err)
assert.Len(t, infos, 3)
analyzerNames := make(map[string]bool)
for _, info := range infos {
assert.Equal(t, "test_field", info.Field)
analyzerNames[info.Name] = true
}
assert.True(t, analyzerNames["default"])
assert.True(t, analyzerNames["english"])
assert.True(t, analyzerNames["chinese"])
})
t.Run("valid params with minimal config", func(t *testing.T) {
coll := createTestCollectionSchema([]*schemapb.FieldSchema{
createTestFieldSchema("string_field", schemapb.DataType_VarChar),
})
fieldSchema := createTestFieldSchema("test_field", schemapb.DataType_VarChar)
infos := make([]*querypb.AnalyzerInfo, 0)
params := `{
"by_field": "string_field",
"analyzers": {"default": {"type": "standard"}}
}`
err := validateMultiAnalyzerParams(params, coll, fieldSchema, &infos)
assert.NoError(t, err)
assert.Len(t, infos, 1)
assert.Equal(t, "default", infos[0].Name)
assert.Equal(t, "test_field", infos[0].Field)
assert.Equal(t, `{"type": "standard"}`, infos[0].Params)
})
}
func Test_validateAnalyzer(t *testing.T) {
createTestCollectionSchemaWithBM25 := func(fields []*schemapb.FieldSchema, inputFieldName string) *schemapb.CollectionSchema {
return &schemapb.CollectionSchema{
Name: "test_collection",
Fields: fields,
Functions: []*schemapb.FunctionSchema{
{
Name: "bm25_func",
Type: schemapb.FunctionType_BM25,
InputFieldNames: []string{inputFieldName},
OutputFieldNames: []string{"bm25_output"},
},
},
}
}
createTestFieldSchema := func(name string, dataType schemapb.DataType, typeParams []*commonpb.KeyValuePair) *schemapb.FieldSchema {
return &schemapb.FieldSchema{
Name: name,
DataType: dataType,
TypeParams: typeParams,
}
}
t.Run("field without enable_match and not BM25 input", func(t *testing.T) {
fieldSchema := createTestFieldSchema("text_field", schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
{Key: common.MaxLengthKey, Value: "100"},
})
collSchema := createTestCollectionSchemaWithBM25([]*schemapb.FieldSchema{fieldSchema}, "invalid_field")
infos := make([]*querypb.AnalyzerInfo, 0)
err := validateAnalyzer(collSchema, fieldSchema, &infos)
assert.NoError(t, err)
assert.Len(t, infos, 0)
})
t.Run("field with enable_match but no enable_analyzer", func(t *testing.T) {
fieldSchema := createTestFieldSchema("text_field", schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
{Key: common.MaxLengthKey, Value: "100"},
{Key: "enable_match", Value: "true"},
})
collSchema := createTestCollectionSchemaWithBM25([]*schemapb.FieldSchema{fieldSchema}, "text_field")
infos := make([]*querypb.AnalyzerInfo, 0)
err := validateAnalyzer(collSchema, fieldSchema, &infos)
assert.Error(t, err)
})
t.Run("field with enable_match and enable_analyzer", func(t *testing.T) {
fieldSchema := createTestFieldSchema("text_field", schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
{Key: common.MaxLengthKey, Value: "100"},
{Key: "enable_match", Value: "true"},
{Key: "enable_analyzer", Value: "true"},
{Key: "analyzer_params", Value: "{\"type\": \"standard\"}"},
})
collSchema := createTestCollectionSchemaWithBM25([]*schemapb.FieldSchema{fieldSchema}, "text_field")
infos := make([]*querypb.AnalyzerInfo, 0)
err := validateAnalyzer(collSchema, fieldSchema, &infos)
assert.NoError(t, err)
assert.Len(t, infos, 1)
assert.Equal(t, "text_field", infos[0].Field)
assert.Equal(t, "{\"type\": \"standard\"}", infos[0].Params)
})
t.Run("field with multi analyzer and enable_match", func(t *testing.T) {
fieldSchema := createTestFieldSchema("text_field", schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
{Key: common.MaxLengthKey, Value: "100"},
{Key: "enable_match", Value: "true"},
{Key: "enable_analyzer", Value: "true"},
{Key: "multi_analyzer_params", Value: `{"by_field": "lang", "analyzers": {"default": "{}"}}`},
})
collSchema := createTestCollectionSchemaWithBM25([]*schemapb.FieldSchema{
fieldSchema,
createTestFieldSchema("lang", schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
{Key: common.MaxLengthKey, Value: "10"},
}),
}, "text_field")
infos := make([]*querypb.AnalyzerInfo, 0)
err := validateAnalyzer(collSchema, fieldSchema, &infos)
assert.Error(t, err)
})
t.Run("field with multi analyzer and analyzer_params", func(t *testing.T) {
fieldSchema := createTestFieldSchema("text_field", schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
{Key: common.MaxLengthKey, Value: "100"},
{Key: "enable_analyzer", Value: "true"},
{Key: "multi_analyzer_params", Value: `{"by_field": "lang", "analyzers": {"default": "{}"}}`},
{Key: "analyzer_params", Value: `{"type": "standard"}`},
})
collSchema := createTestCollectionSchemaWithBM25([]*schemapb.FieldSchema{
fieldSchema,
createTestFieldSchema("lang", schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
{Key: common.MaxLengthKey, Value: "10"},
}),
}, "text_field")
infos := make([]*querypb.AnalyzerInfo, 0)
err := validateAnalyzer(collSchema, fieldSchema, &infos)
assert.Error(t, err)
})
t.Run("field with valid multi analyzer", func(t *testing.T) {
// Create a field with valid multi analyzer
fieldSchema := createTestFieldSchema("text_field", schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
{Key: common.MaxLengthKey, Value: "100"},
{Key: "enable_analyzer", Value: "true"},
{Key: "multi_analyzer_params", Value: `{
"by_field": "lang",
"analyzers": {
"default": {"type": "standard"},
"english": {"type": "english"}
}
}`},
})
collSchema := createTestCollectionSchemaWithBM25([]*schemapb.FieldSchema{
fieldSchema,
createTestFieldSchema("lang", schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
{Key: common.MaxLengthKey, Value: "10"},
}),
}, "text_field")
infos := make([]*querypb.AnalyzerInfo, 0)
err := validateAnalyzer(collSchema, fieldSchema, &infos)
assert.NoError(t, err)
assert.Len(t, infos, 2)
// Verify analyzer info content
analyzerNames := make(map[string]bool)
for _, info := range infos {
assert.Equal(t, "text_field", info.Field)
analyzerNames[info.Name] = true
}
assert.True(t, analyzerNames["default"])
assert.True(t, analyzerNames["english"])
})
t.Run("field with invalid multi analyzer params", func(t *testing.T) {
// Create a field with invalid multi analyzer params
fieldSchema := createTestFieldSchema("text_field", schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
{Key: common.MaxLengthKey, Value: "100"},
{Key: "enable_analyzer", Value: "true"},
{Key: "multi_analyzer_params", Value: `{"by_field": "non_existent_field", "analyzers": {"default": "{}"}}`},
})
collSchema := createTestCollectionSchemaWithBM25([]*schemapb.FieldSchema{fieldSchema}, "text_field")
infos := make([]*querypb.AnalyzerInfo, 0)
err := validateAnalyzer(collSchema, fieldSchema, &infos)
assert.Error(t, err)
})
t.Run("field with analyzer_params only", func(t *testing.T) {
// Create a field with analyzer_params only
fieldSchema := createTestFieldSchema("text_field", schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
{Key: common.MaxLengthKey, Value: "100"},
{Key: "enable_analyzer", Value: "true"},
{Key: "analyzer_params", Value: `{"type": "standard"}`},
})
collSchema := createTestCollectionSchemaWithBM25([]*schemapb.FieldSchema{fieldSchema}, "text_field")
infos := make([]*querypb.AnalyzerInfo, 0)
err := validateAnalyzer(collSchema, fieldSchema, &infos)
require.NoError(t, err)
require.Len(t, infos, 1)
assert.Equal(t, "text_field", infos[0].Field)
assert.Equal(t, "", infos[0].Name) // Regular analyzer has no name
assert.Equal(t, `{"type": "standard"}`, infos[0].Params)
})
t.Run("field with enable_analyzer but no analyzer_params", func(t *testing.T) {
// Create a field with enable_analyzer but no analyzer_params (uses default analyzer)
fieldSchema := createTestFieldSchema("text_field", schemapb.DataType_VarChar, []*commonpb.KeyValuePair{
{Key: common.MaxLengthKey, Value: "100"},
{Key: "enable_match", Value: "true"},
{Key: "enable_analyzer", Value: "true"},
})
collSchema := createTestCollectionSchemaWithBM25([]*schemapb.FieldSchema{fieldSchema}, "text_field")
infos := make([]*querypb.AnalyzerInfo, 0)
err := validateAnalyzer(collSchema, fieldSchema, &infos)
assert.NoError(t, err)
assert.Len(t, infos, 0) // No analyzer_params, uses default analyzer
})
}

View File

@ -398,7 +398,7 @@ func RowBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *schemap
} }
for _, field := range collSchema.Fields { for _, field := range collSchema.Fields {
if skipFunction && IsBM25FunctionOutputField(field, collSchema) { if skipFunction && typeutil.IsBM25FunctionOutputField(field, collSchema) {
continue continue
} }
@ -777,7 +777,7 @@ func ColumnBasedInsertMsgToInsertData(msg *msgstream.InsertMsg, collSchema *sche
} }
handleFieldData := func(field *schemapb.FieldSchema) (FieldData, error) { handleFieldData := func(field *schemapb.FieldSchema) (FieldData, error) {
if IsBM25FunctionOutputField(field, collSchema) { if typeutil.IsBM25FunctionOutputField(field, collSchema) {
return nil, nil return nil, nil
} }
@ -1562,25 +1562,6 @@ func (ni NullableInt) IsNull() bool {
return ni.Value == nil return ni.Value == nil
} }
// TODO: unify the function implementation, storage/utils.go & proxy/util.go
func IsBM25FunctionOutputField(field *schemapb.FieldSchema, collSchema *schemapb.CollectionSchema) bool {
if !(field.GetIsFunctionOutput() && field.GetDataType() == schemapb.DataType_SparseFloatVector) {
return false
}
for _, fSchema := range collSchema.Functions {
if fSchema.Type == schemapb.FunctionType_BM25 {
if len(fSchema.OutputFieldNames) != 0 && field.Name == fSchema.OutputFieldNames[0] {
return true
}
if len(fSchema.OutputFieldIds) != 0 && field.FieldID == fSchema.OutputFieldIds[0] {
return true
}
}
}
return false
}
func GetDefaultValue(fieldSchema *schemapb.FieldSchema) interface{} { func GetDefaultValue(fieldSchema *schemapb.FieldSchema) interface{} {
switch fieldSchema.DataType { switch fieldSchema.DataType {
case schemapb.DataType_Bool: case schemapb.DataType_Bool:

View File

@ -2075,7 +2075,7 @@ func TestBM25Checker(t *testing.T) {
} }
for _, field := range schema.Fields { for _, field := range schema.Fields {
isBm25 := IsBM25FunctionOutputField(field, schema) isBm25 := typeutil.IsBM25FunctionOutputField(field, schema)
if field.FieldID == 103 { if field.FieldID == 103 {
assert.True(t, isBm25) assert.True(t, isBm25)
} else { } else {

View File

@ -193,3 +193,11 @@ func (m *GrpcQueryCoordClient) UpdateLoadConfig(ctx context.Context, req *queryp
func (m *GrpcQueryCoordClient) ListLoadedSegments(ctx context.Context, req *querypb.ListLoadedSegmentsRequest, opts ...grpc.CallOption) (*querypb.ListLoadedSegmentsResponse, error) { func (m *GrpcQueryCoordClient) ListLoadedSegments(ctx context.Context, req *querypb.ListLoadedSegmentsRequest, opts ...grpc.CallOption) (*querypb.ListLoadedSegmentsResponse, error) {
return &querypb.ListLoadedSegmentsResponse{}, m.Err return &querypb.ListLoadedSegmentsResponse{}, m.Err
} }
func (m *GrpcQueryCoordClient) RunAnalyzer(ctx context.Context, req *querypb.RunAnalyzerRequest, opts ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error) {
return &milvuspb.RunAnalyzerResponse{}, m.Err
}
func (m *GrpcQueryCoordClient) ValidateAnalyzer(ctx context.Context, req *querypb.ValidateAnalyzerRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
return &commonpb.Status{}, m.Err
}

View File

@ -142,6 +142,10 @@ func (m *GrpcQueryNodeClient) RunAnalyzer(ctx context.Context, in *querypb.RunAn
return &milvuspb.RunAnalyzerResponse{}, m.Err return &milvuspb.RunAnalyzerResponse{}, m.Err
} }
func (m *GrpcQueryNodeClient) ValidateAnalyzer(ctx context.Context, in *querypb.ValidateAnalyzerRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
return &commonpb.Status{}, m.Err
}
func (m *GrpcQueryNodeClient) Close() error { func (m *GrpcQueryNodeClient) Close() error {
return m.Err return m.Err
} }

View File

@ -164,6 +164,10 @@ func (qn *qnServerWrapper) DropIndex(ctx context.Context, in *querypb.DropIndexR
return qn.QueryNode.DropIndex(ctx, in) return qn.QueryNode.DropIndex(ctx, in)
} }
func (qn *qnServerWrapper) ValidateAnalyzer(ctx context.Context, in *querypb.ValidateAnalyzerRequest, _ ...grpc.CallOption) (*commonpb.Status, error) {
return qn.QueryNode.ValidateAnalyzer(ctx, in)
}
func WrapQueryNodeServerAsClient(qn types.QueryNode) types.QueryNodeClient { func WrapQueryNodeServerAsClient(qn types.QueryNode) types.QueryNodeClient {
return &qnServerWrapper{ return &qnServerWrapper{
QueryNode: qn, QueryNode: qn,

View File

@ -100,6 +100,8 @@ service QueryCoord {
rpc CheckQueryNodeDistribution(CheckQueryNodeDistributionRequest) returns (common.Status) {} rpc CheckQueryNodeDistribution(CheckQueryNodeDistributionRequest) returns (common.Status) {}
rpc UpdateLoadConfig(UpdateLoadConfigRequest) returns (common.Status) {} rpc UpdateLoadConfig(UpdateLoadConfigRequest) returns (common.Status) {}
rpc RunAnalyzer(RunAnalyzerRequest) returns(milvus.RunAnalyzerResponse){}
rpc ValidateAnalyzer(ValidateAnalyzerRequest) returns(common.Status){}
} }
service QueryNode { service QueryNode {
@ -173,6 +175,7 @@ service QueryNode {
rpc RunAnalyzer(RunAnalyzerRequest) returns(milvus.RunAnalyzerResponse){} rpc RunAnalyzer(RunAnalyzerRequest) returns(milvus.RunAnalyzerResponse){}
rpc DropIndex(DropIndexRequest) returns (common.Status) {} rpc DropIndex(DropIndexRequest) returns (common.Status) {}
rpc ValidateAnalyzer(ValidateAnalyzerRequest) returns(common.Status){}
} }
// --------------------QueryCoord grpc request and response proto------------------ // --------------------QueryCoord grpc request and response proto------------------
@ -993,6 +996,19 @@ message RunAnalyzerRequest{
bool with_detail = 6; bool with_detail = 6;
bool with_hash = 7; bool with_hash = 7;
string analyzer_params = 8;
}
message AnalyzerInfo{
string params = 1;
string field = 2;
string name = 3;
}
message ValidateAnalyzerRequest{
common.MsgBase base = 1;
repeated AnalyzerInfo analyzer_infos = 2;
} }
message ListLoadedSegmentsRequest { message ListLoadedSegmentsRequest {

File diff suppressed because it is too large Load Diff

View File

@ -59,6 +59,8 @@ const (
QueryCoord_TransferChannel_FullMethodName = "/milvus.proto.query.QueryCoord/TransferChannel" QueryCoord_TransferChannel_FullMethodName = "/milvus.proto.query.QueryCoord/TransferChannel"
QueryCoord_CheckQueryNodeDistribution_FullMethodName = "/milvus.proto.query.QueryCoord/CheckQueryNodeDistribution" QueryCoord_CheckQueryNodeDistribution_FullMethodName = "/milvus.proto.query.QueryCoord/CheckQueryNodeDistribution"
QueryCoord_UpdateLoadConfig_FullMethodName = "/milvus.proto.query.QueryCoord/UpdateLoadConfig" QueryCoord_UpdateLoadConfig_FullMethodName = "/milvus.proto.query.QueryCoord/UpdateLoadConfig"
QueryCoord_RunAnalyzer_FullMethodName = "/milvus.proto.query.QueryCoord/RunAnalyzer"
QueryCoord_ValidateAnalyzer_FullMethodName = "/milvus.proto.query.QueryCoord/ValidateAnalyzer"
) )
// QueryCoordClient is the client API for QueryCoord service. // QueryCoordClient is the client API for QueryCoord service.
@ -106,6 +108,8 @@ type QueryCoordClient interface {
TransferChannel(ctx context.Context, in *TransferChannelRequest, opts ...grpc.CallOption) (*commonpb.Status, error) TransferChannel(ctx context.Context, in *TransferChannelRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
CheckQueryNodeDistribution(ctx context.Context, in *CheckQueryNodeDistributionRequest, opts ...grpc.CallOption) (*commonpb.Status, error) CheckQueryNodeDistribution(ctx context.Context, in *CheckQueryNodeDistributionRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
UpdateLoadConfig(ctx context.Context, in *UpdateLoadConfigRequest, opts ...grpc.CallOption) (*commonpb.Status, error) UpdateLoadConfig(ctx context.Context, in *UpdateLoadConfigRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
RunAnalyzer(ctx context.Context, in *RunAnalyzerRequest, opts ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error)
ValidateAnalyzer(ctx context.Context, in *ValidateAnalyzerRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
} }
type queryCoordClient struct { type queryCoordClient struct {
@ -450,6 +454,24 @@ func (c *queryCoordClient) UpdateLoadConfig(ctx context.Context, in *UpdateLoadC
return out, nil return out, nil
} }
func (c *queryCoordClient) RunAnalyzer(ctx context.Context, in *RunAnalyzerRequest, opts ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error) {
out := new(milvuspb.RunAnalyzerResponse)
err := c.cc.Invoke(ctx, QueryCoord_RunAnalyzer_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *queryCoordClient) ValidateAnalyzer(ctx context.Context, in *ValidateAnalyzerRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
out := new(commonpb.Status)
err := c.cc.Invoke(ctx, QueryCoord_ValidateAnalyzer_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// QueryCoordServer is the server API for QueryCoord service. // QueryCoordServer is the server API for QueryCoord service.
// All implementations should embed UnimplementedQueryCoordServer // All implementations should embed UnimplementedQueryCoordServer
// for forward compatibility // for forward compatibility
@ -495,6 +517,8 @@ type QueryCoordServer interface {
TransferChannel(context.Context, *TransferChannelRequest) (*commonpb.Status, error) TransferChannel(context.Context, *TransferChannelRequest) (*commonpb.Status, error)
CheckQueryNodeDistribution(context.Context, *CheckQueryNodeDistributionRequest) (*commonpb.Status, error) CheckQueryNodeDistribution(context.Context, *CheckQueryNodeDistributionRequest) (*commonpb.Status, error)
UpdateLoadConfig(context.Context, *UpdateLoadConfigRequest) (*commonpb.Status, error) UpdateLoadConfig(context.Context, *UpdateLoadConfigRequest) (*commonpb.Status, error)
RunAnalyzer(context.Context, *RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error)
ValidateAnalyzer(context.Context, *ValidateAnalyzerRequest) (*commonpb.Status, error)
} }
// UnimplementedQueryCoordServer should be embedded to have forward compatible implementations. // UnimplementedQueryCoordServer should be embedded to have forward compatible implementations.
@ -612,6 +636,12 @@ func (UnimplementedQueryCoordServer) CheckQueryNodeDistribution(context.Context,
func (UnimplementedQueryCoordServer) UpdateLoadConfig(context.Context, *UpdateLoadConfigRequest) (*commonpb.Status, error) { func (UnimplementedQueryCoordServer) UpdateLoadConfig(context.Context, *UpdateLoadConfigRequest) (*commonpb.Status, error) {
return nil, status.Errorf(codes.Unimplemented, "method UpdateLoadConfig not implemented") return nil, status.Errorf(codes.Unimplemented, "method UpdateLoadConfig not implemented")
} }
func (UnimplementedQueryCoordServer) RunAnalyzer(context.Context, *RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) {
return nil, status.Errorf(codes.Unimplemented, "method RunAnalyzer not implemented")
}
func (UnimplementedQueryCoordServer) ValidateAnalyzer(context.Context, *ValidateAnalyzerRequest) (*commonpb.Status, error) {
return nil, status.Errorf(codes.Unimplemented, "method ValidateAnalyzer not implemented")
}
// UnsafeQueryCoordServer may be embedded to opt out of forward compatibility for this service. // UnsafeQueryCoordServer may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to QueryCoordServer will // Use of this interface is not recommended, as added methods to QueryCoordServer will
@ -1290,6 +1320,42 @@ func _QueryCoord_UpdateLoadConfig_Handler(srv interface{}, ctx context.Context,
return interceptor(ctx, in, info, handler) return interceptor(ctx, in, info, handler)
} }
func _QueryCoord_RunAnalyzer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(RunAnalyzerRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(QueryCoordServer).RunAnalyzer(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: QueryCoord_RunAnalyzer_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(QueryCoordServer).RunAnalyzer(ctx, req.(*RunAnalyzerRequest))
}
return interceptor(ctx, in, info, handler)
}
func _QueryCoord_ValidateAnalyzer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ValidateAnalyzerRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(QueryCoordServer).ValidateAnalyzer(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: QueryCoord_ValidateAnalyzer_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(QueryCoordServer).ValidateAnalyzer(ctx, req.(*ValidateAnalyzerRequest))
}
return interceptor(ctx, in, info, handler)
}
// QueryCoord_ServiceDesc is the grpc.ServiceDesc for QueryCoord service. // QueryCoord_ServiceDesc is the grpc.ServiceDesc for QueryCoord service.
// It's only intended for direct use with grpc.RegisterService, // It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy) // and not to be introspected or modified (even as a copy)
@ -1445,6 +1511,14 @@ var QueryCoord_ServiceDesc = grpc.ServiceDesc{
MethodName: "UpdateLoadConfig", MethodName: "UpdateLoadConfig",
Handler: _QueryCoord_UpdateLoadConfig_Handler, Handler: _QueryCoord_UpdateLoadConfig_Handler,
}, },
{
MethodName: "RunAnalyzer",
Handler: _QueryCoord_RunAnalyzer_Handler,
},
{
MethodName: "ValidateAnalyzer",
Handler: _QueryCoord_ValidateAnalyzer_Handler,
},
}, },
Streams: []grpc.StreamDesc{}, Streams: []grpc.StreamDesc{},
Metadata: "query_coord.proto", Metadata: "query_coord.proto",
@ -1479,6 +1553,7 @@ const (
QueryNode_UpdateSchema_FullMethodName = "/milvus.proto.query.QueryNode/UpdateSchema" QueryNode_UpdateSchema_FullMethodName = "/milvus.proto.query.QueryNode/UpdateSchema"
QueryNode_RunAnalyzer_FullMethodName = "/milvus.proto.query.QueryNode/RunAnalyzer" QueryNode_RunAnalyzer_FullMethodName = "/milvus.proto.query.QueryNode/RunAnalyzer"
QueryNode_DropIndex_FullMethodName = "/milvus.proto.query.QueryNode/DropIndex" QueryNode_DropIndex_FullMethodName = "/milvus.proto.query.QueryNode/DropIndex"
QueryNode_ValidateAnalyzer_FullMethodName = "/milvus.proto.query.QueryNode/ValidateAnalyzer"
) )
// QueryNodeClient is the client API for QueryNode service. // QueryNodeClient is the client API for QueryNode service.
@ -1516,6 +1591,7 @@ type QueryNodeClient interface {
UpdateSchema(ctx context.Context, in *UpdateSchemaRequest, opts ...grpc.CallOption) (*commonpb.Status, error) UpdateSchema(ctx context.Context, in *UpdateSchemaRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
RunAnalyzer(ctx context.Context, in *RunAnalyzerRequest, opts ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error) RunAnalyzer(ctx context.Context, in *RunAnalyzerRequest, opts ...grpc.CallOption) (*milvuspb.RunAnalyzerResponse, error)
DropIndex(ctx context.Context, in *DropIndexRequest, opts ...grpc.CallOption) (*commonpb.Status, error) DropIndex(ctx context.Context, in *DropIndexRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
ValidateAnalyzer(ctx context.Context, in *ValidateAnalyzerRequest, opts ...grpc.CallOption) (*commonpb.Status, error)
} }
type queryNodeClient struct { type queryNodeClient struct {
@ -1824,6 +1900,15 @@ func (c *queryNodeClient) DropIndex(ctx context.Context, in *DropIndexRequest, o
return out, nil return out, nil
} }
func (c *queryNodeClient) ValidateAnalyzer(ctx context.Context, in *ValidateAnalyzerRequest, opts ...grpc.CallOption) (*commonpb.Status, error) {
out := new(commonpb.Status)
err := c.cc.Invoke(ctx, QueryNode_ValidateAnalyzer_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
// QueryNodeServer is the server API for QueryNode service. // QueryNodeServer is the server API for QueryNode service.
// All implementations should embed UnimplementedQueryNodeServer // All implementations should embed UnimplementedQueryNodeServer
// for forward compatibility // for forward compatibility
@ -1859,6 +1944,7 @@ type QueryNodeServer interface {
UpdateSchema(context.Context, *UpdateSchemaRequest) (*commonpb.Status, error) UpdateSchema(context.Context, *UpdateSchemaRequest) (*commonpb.Status, error)
RunAnalyzer(context.Context, *RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error) RunAnalyzer(context.Context, *RunAnalyzerRequest) (*milvuspb.RunAnalyzerResponse, error)
DropIndex(context.Context, *DropIndexRequest) (*commonpb.Status, error) DropIndex(context.Context, *DropIndexRequest) (*commonpb.Status, error)
ValidateAnalyzer(context.Context, *ValidateAnalyzerRequest) (*commonpb.Status, error)
} }
// UnimplementedQueryNodeServer should be embedded to have forward compatible implementations. // UnimplementedQueryNodeServer should be embedded to have forward compatible implementations.
@ -1949,6 +2035,9 @@ func (UnimplementedQueryNodeServer) RunAnalyzer(context.Context, *RunAnalyzerReq
func (UnimplementedQueryNodeServer) DropIndex(context.Context, *DropIndexRequest) (*commonpb.Status, error) { func (UnimplementedQueryNodeServer) DropIndex(context.Context, *DropIndexRequest) (*commonpb.Status, error) {
return nil, status.Errorf(codes.Unimplemented, "method DropIndex not implemented") return nil, status.Errorf(codes.Unimplemented, "method DropIndex not implemented")
} }
func (UnimplementedQueryNodeServer) ValidateAnalyzer(context.Context, *ValidateAnalyzerRequest) (*commonpb.Status, error) {
return nil, status.Errorf(codes.Unimplemented, "method ValidateAnalyzer not implemented")
}
// UnsafeQueryNodeServer may be embedded to opt out of forward compatibility for this service. // UnsafeQueryNodeServer may be embedded to opt out of forward compatibility for this service.
// Use of this interface is not recommended, as added methods to QueryNodeServer will // Use of this interface is not recommended, as added methods to QueryNodeServer will
@ -2471,6 +2560,24 @@ func _QueryNode_DropIndex_Handler(srv interface{}, ctx context.Context, dec func
return interceptor(ctx, in, info, handler) return interceptor(ctx, in, info, handler)
} }
func _QueryNode_ValidateAnalyzer_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(ValidateAnalyzerRequest)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(QueryNodeServer).ValidateAnalyzer(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: QueryNode_ValidateAnalyzer_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(QueryNodeServer).ValidateAnalyzer(ctx, req.(*ValidateAnalyzerRequest))
}
return interceptor(ctx, in, info, handler)
}
// QueryNode_ServiceDesc is the grpc.ServiceDesc for QueryNode service. // QueryNode_ServiceDesc is the grpc.ServiceDesc for QueryNode service.
// It's only intended for direct use with grpc.RegisterService, // It's only intended for direct use with grpc.RegisterService,
// and not to be introspected or modified (even as a copy) // and not to be introspected or modified (even as a copy)
@ -2582,6 +2689,10 @@ var QueryNode_ServiceDesc = grpc.ServiceDesc{
MethodName: "DropIndex", MethodName: "DropIndex",
Handler: _QueryNode_DropIndex_Handler, Handler: _QueryNode_DropIndex_Handler,
}, },
{
MethodName: "ValidateAnalyzer",
Handler: _QueryNode_ValidateAnalyzer_Handler,
},
}, },
Streams: []grpc.StreamDesc{ Streams: []grpc.StreamDesc{
{ {

View File

@ -43,7 +43,7 @@ func Init() error {
exp, err := CreateTracerExporter(params) exp, err := CreateTracerExporter(params)
if err != nil { if err != nil {
log.Warn("Init tracer faield", zap.Error(err)) log.Warn("Init tracer failed", zap.Error(err))
return err return err
} }

View File

@ -199,12 +199,12 @@ func TestGetCollectionIDFromVChannel(t *testing.T) {
collectionID := GetCollectionIDFromVChannel(vChannel1) collectionID := GetCollectionIDFromVChannel(vChannel1)
assert.Equal(t, int64(449684528748778322), collectionID) assert.Equal(t, int64(449684528748778322), collectionID)
invailedVChannel := "06b84fe16780ed1-rootcoord-dm_3_v0" invaildVChannel := "06b84fe16780ed1-rootcoord-dm_3_v0"
collectionID = GetCollectionIDFromVChannel(invailedVChannel) collectionID = GetCollectionIDFromVChannel(invaildVChannel)
assert.Equal(t, int64(-1), collectionID) assert.Equal(t, int64(-1), collectionID)
invailedVChannel = "06b84fe16780ed1-rootcoord-dm_3_-1v0" invaildVChannel = "06b84fe16780ed1-rootcoord-dm_3_-1v0"
collectionID = GetCollectionIDFromVChannel(invailedVChannel) collectionID = GetCollectionIDFromVChannel(invaildVChannel)
assert.Equal(t, int64(-1), collectionID) assert.Equal(t, int64(-1), collectionID)
} }

View File

@ -80,9 +80,6 @@ func (h *FieldSchemaHelper) GetMultiAnalyzerParams() (string, bool) {
} }
func (h *FieldSchemaHelper) HasAnalyzerParams() bool { func (h *FieldSchemaHelper) HasAnalyzerParams() bool {
if !IsStringType(h.schema.GetDataType()) {
return false
}
_, err := h.typeParams.Get("analyzer_params") _, err := h.typeParams.Get("analyzer_params")
return err == nil return err == nil
} }

View File

@ -2429,3 +2429,30 @@ func GetNeedProcessFunctions(fieldIDs []int64, functions []*schemapb.FunctionSch
} }
return needProcessFunctions, nil return needProcessFunctions, nil
} }
func IsBM25FunctionOutputField(field *schemapb.FieldSchema, collSchema *schemapb.CollectionSchema) bool {
if !(field.GetIsFunctionOutput() && field.GetDataType() == schemapb.DataType_SparseFloatVector) {
return false
}
for _, fSchema := range collSchema.Functions {
if fSchema.Type == schemapb.FunctionType_BM25 {
if len(fSchema.OutputFieldNames) != 0 && field.Name == fSchema.OutputFieldNames[0] {
return true
}
if len(fSchema.OutputFieldIds) != 0 && field.FieldID == fSchema.OutputFieldIds[0] {
return true
}
}
}
return false
}
func IsBm25FunctionInputField(coll *schemapb.CollectionSchema, field *schemapb.FieldSchema) bool {
for _, fn := range coll.GetFunctions() {
if fn.GetType() == schemapb.FunctionType_BM25 && field.GetName() == fn.GetInputFieldNames()[0] {
return true
}
}
return false
}

View File

@ -4724,3 +4724,257 @@ func TestUpdateFieldData_GeometryAndTimestamptz(t *testing.T) {
assert.Equal(t, []byte{0xFB, 0xFA}, geoData[2]) assert.Equal(t, []byte{0xFB, 0xFA}, geoData[2])
}) })
} }
func TestIsBM25FunctionOutputField(t *testing.T) {
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{Name: "input_field", DataType: schemapb.DataType_VarChar, TypeParams: []*commonpb.KeyValuePair{{Key: "enable_analyzer", Value: "true"}}},
{Name: "output_field", DataType: schemapb.DataType_SparseFloatVector, IsFunctionOutput: true},
},
Functions: []*schemapb.FunctionSchema{
{
Name: "bm25_func",
Type: schemapb.FunctionType_BM25,
InputFieldNames: []string{"input_field"},
OutputFieldNames: []string{"output_field"},
},
},
}
assert.False(t, IsBM25FunctionOutputField(schema.Fields[0], schema))
assert.True(t, IsBM25FunctionOutputField(schema.Fields[1], schema))
// Test with field IDs instead of names
schemaWithIds := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{Name: "input_field", FieldID: 100, DataType: schemapb.DataType_VarChar, TypeParams: []*commonpb.KeyValuePair{{Key: "enable_analyzer", Value: "true"}}},
{Name: "output_field", FieldID: 101, DataType: schemapb.DataType_SparseFloatVector, IsFunctionOutput: true},
},
Functions: []*schemapb.FunctionSchema{
{
Name: "bm25_func",
Type: schemapb.FunctionType_BM25,
InputFieldIds: []int64{100},
OutputFieldIds: []int64{101},
},
},
}
assert.False(t, IsBM25FunctionOutputField(schemaWithIds.Fields[0], schemaWithIds))
assert.True(t, IsBM25FunctionOutputField(schemaWithIds.Fields[1], schemaWithIds))
// Test with non-BM25 function
nonBM25Schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{Name: "input_field", DataType: schemapb.DataType_VarChar},
{Name: "output_field", DataType: schemapb.DataType_SparseFloatVector, IsFunctionOutput: true},
},
Functions: []*schemapb.FunctionSchema{
{
Name: "other_func",
Type: schemapb.FunctionType_Unknown,
InputFieldNames: []string{"input_field"},
OutputFieldNames: []string{"output_field"},
},
},
}
assert.False(t, IsBM25FunctionOutputField(nonBM25Schema.Fields[1], nonBM25Schema))
// Test with non-sparse vector field
nonSparseSchema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{Name: "input_field", DataType: schemapb.DataType_VarChar},
{Name: "output_field", DataType: schemapb.DataType_FloatVector, IsFunctionOutput: true},
},
Functions: []*schemapb.FunctionSchema{
{
Name: "bm25_func",
Type: schemapb.FunctionType_BM25,
InputFieldNames: []string{"input_field"},
OutputFieldNames: []string{"output_field"},
},
},
}
assert.False(t, IsBM25FunctionOutputField(nonSparseSchema.Fields[1], nonSparseSchema))
// Test with field not marked as function output
nonFunctionOutputSchema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{Name: "input_field", DataType: schemapb.DataType_VarChar},
{Name: "output_field", DataType: schemapb.DataType_SparseFloatVector, IsFunctionOutput: false},
},
Functions: []*schemapb.FunctionSchema{
{
Name: "bm25_func",
Type: schemapb.FunctionType_BM25,
InputFieldNames: []string{"input_field"},
OutputFieldNames: []string{"output_field"},
},
},
}
assert.False(t, IsBM25FunctionOutputField(nonFunctionOutputSchema.Fields[1], nonFunctionOutputSchema))
}
func TestIsBm25FunctionInputField(t *testing.T) {
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{Name: "input_field", DataType: schemapb.DataType_VarChar, TypeParams: []*commonpb.KeyValuePair{{Key: "enable_analyzer", Value: "true"}}},
{Name: "output_field", DataType: schemapb.DataType_SparseFloatVector, IsFunctionOutput: true},
},
Functions: []*schemapb.FunctionSchema{
{
Name: "bm25_func",
Type: schemapb.FunctionType_BM25,
InputFieldNames: []string{"input_field"},
OutputFieldNames: []string{"output_field"},
},
},
}
assert.True(t, IsBm25FunctionInputField(schema, schema.Fields[0]))
assert.False(t, IsBm25FunctionInputField(schema, schema.Fields[1]))
// Test with multiple functions, only one is BM25
multipleSchema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{Name: "input_field1", DataType: schemapb.DataType_VarChar},
{Name: "input_field2", DataType: schemapb.DataType_VarChar},
{Name: "output_field1", DataType: schemapb.DataType_SparseFloatVector, IsFunctionOutput: true},
{Name: "output_field2", DataType: schemapb.DataType_FloatVector, IsFunctionOutput: true},
},
Functions: []*schemapb.FunctionSchema{
{
Name: "bm25_func",
Type: schemapb.FunctionType_BM25,
InputFieldNames: []string{"input_field1"},
OutputFieldNames: []string{"output_field1"},
},
{
Name: "other_func",
Type: schemapb.FunctionType_Unknown,
InputFieldNames: []string{"input_field2"},
OutputFieldNames: []string{"output_field2"},
},
},
}
assert.True(t, IsBm25FunctionInputField(multipleSchema, multipleSchema.Fields[0]))
assert.False(t, IsBm25FunctionInputField(multipleSchema, multipleSchema.Fields[1]))
assert.False(t, IsBm25FunctionInputField(multipleSchema, multipleSchema.Fields[2]))
assert.False(t, IsBm25FunctionInputField(multipleSchema, multipleSchema.Fields[3]))
// Test with no BM25 functions
noBM25Schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{Name: "input_field", DataType: schemapb.DataType_VarChar},
{Name: "output_field", DataType: schemapb.DataType_FloatVector, IsFunctionOutput: true},
},
Functions: []*schemapb.FunctionSchema{
{
Name: "other_func",
Type: schemapb.FunctionType_Unknown,
InputFieldNames: []string{"input_field"},
OutputFieldNames: []string{"output_field"},
},
},
}
assert.False(t, IsBm25FunctionInputField(noBM25Schema, noBM25Schema.Fields[0]))
// Test with empty functions
emptySchema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{Name: "input_field", DataType: schemapb.DataType_VarChar},
},
Functions: []*schemapb.FunctionSchema{},
}
assert.False(t, IsBm25FunctionInputField(emptySchema, emptySchema.Fields[0]))
// Test with nil functions
nilSchema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{Name: "input_field", DataType: schemapb.DataType_VarChar},
},
Functions: nil,
}
assert.False(t, IsBm25FunctionInputField(nilSchema, nilSchema.Fields[0]))
}
func TestSchemaHelper_GetFunctionByOutputField(t *testing.T) {
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{Name: "input_field", FieldID: 100, DataType: schemapb.DataType_VarChar},
{Name: "output_field", FieldID: 101, DataType: schemapb.DataType_SparseFloatVector, IsFunctionOutput: true},
{Name: "regular_field", FieldID: 102, DataType: schemapb.DataType_Int64},
},
Functions: []*schemapb.FunctionSchema{
{
Name: "bm25_func",
Type: schemapb.FunctionType_BM25,
InputFieldIds: []int64{100},
OutputFieldIds: []int64{101},
},
},
}
helper, err := CreateSchemaHelper(schema)
assert.NoError(t, err)
// Test getting function by output field
function, err := helper.GetFunctionByOutputField(schema.Fields[1])
assert.NoError(t, err)
assert.Equal(t, "bm25_func", function.Name)
assert.Equal(t, schemapb.FunctionType_BM25, function.Type)
// Test with non-function output field
_, err = helper.GetFunctionByOutputField(schema.Fields[0])
assert.Error(t, err)
assert.Contains(t, err.Error(), "function not exist")
// Test with regular field
_, err = helper.GetFunctionByOutputField(schema.Fields[2])
assert.Error(t, err)
assert.Contains(t, err.Error(), "function not exist")
}
func TestSchemaHelper_CanRetrieveRawFieldData(t *testing.T) {
schema := &schemapb.CollectionSchema{
Fields: []*schemapb.FieldSchema{
{Name: "input_field", FieldID: 100, DataType: schemapb.DataType_VarChar},
{Name: "bm25_output", FieldID: 101, DataType: schemapb.DataType_SparseFloatVector, IsFunctionOutput: true},
{Name: "other_output", FieldID: 102, DataType: schemapb.DataType_FloatVector, IsFunctionOutput: true},
{Name: "regular_field", FieldID: 103, DataType: schemapb.DataType_Int64},
},
Functions: []*schemapb.FunctionSchema{
{
Name: "bm25_func",
Type: schemapb.FunctionType_BM25,
InputFieldIds: []int64{100},
OutputFieldIds: []int64{101},
},
{
Name: "other_func",
Type: schemapb.FunctionType_Unknown,
InputFieldIds: []int64{100},
OutputFieldIds: []int64{102},
},
},
}
helper, err := CreateSchemaHelper(schema)
assert.NoError(t, err)
// Regular field should be retrievable
assert.True(t, helper.CanRetrieveRawFieldData(schema.Fields[0]))
assert.True(t, helper.CanRetrieveRawFieldData(schema.Fields[3]))
// BM25 function output field should NOT be retrievable
assert.False(t, helper.CanRetrieveRawFieldData(schema.Fields[1]))
// Other function output field should be retrievable
assert.True(t, helper.CanRetrieveRawFieldData(schema.Fields[2]))
// Test with field that has IsFunctionOutput=true but no corresponding function
orphanField := &schemapb.FieldSchema{
Name: "orphan_field",
FieldID: 104,
DataType: schemapb.DataType_FloatVector,
IsFunctionOutput: true,
}
assert.False(t, helper.CanRetrieveRawFieldData(orphanField))
}