diff --git a/go.mod b/go.mod index 70bacce52e..d439c52811 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/klauspost/compress v1.17.9 github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d - github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250107132849-4099923438e6 + github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250208062437-5af22aa4b559 github.com/minio/minio-go/v7 v7.0.73 github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81 github.com/prometheus/client_golang v1.14.0 diff --git a/go.sum b/go.sum index a2a0e9a243..352cefd651 100644 --- a/go.sum +++ b/go.sum @@ -660,8 +660,8 @@ github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119 h1:9VXijWu github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg= github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8= github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4= -github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250107132849-4099923438e6 h1:IUHMoxffuI2xbtMJJE/XK7kKzavZnN5sMiheIKTDVt8= -github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250107132849-4099923438e6/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs= +github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250208062437-5af22aa4b559 h1:c8n10eBkYU/HYaDUNAaKog4aIA3ZHO+GL7bHN2Ug/MA= +github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250208062437-5af22aa4b559/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs= github.com/milvus-io/pulsar-client-go v0.12.1 h1:O2JZp1tsYiO7C0MQ4hrUY/aJXnn2Gry6hpm7UodghmE= github.com/milvus-io/pulsar-client-go v0.12.1/go.mod h1:dkutuH4oS2pXiGm+Ti7fQZ4MRjrMPZ8IJeEGAWMeckk= github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs= diff --git a/internal/distributed/proxy/service.go b/internal/distributed/proxy/service.go index c2f2bea06c..b1d5c5b97b 100644 --- a/internal/distributed/proxy/service.go +++ b/internal/distributed/proxy/service.go @@ -1157,3 +1157,7 @@ func (s *Server) InvalidateShardLeaderCache(ctx context.Context, req *proxypb.In func (s *Server) DescribeDatabase(ctx context.Context, req *milvuspb.DescribeDatabaseRequest) (*milvuspb.DescribeDatabaseResponse, error) { return s.proxy.DescribeDatabase(ctx, req) } + +func (s *Server) RunAnalyzer(ctx context.Context, req *milvuspb.RunAnalyzerRequset) (*milvuspb.RunAnalyzerResponse, error) { + return s.proxy.RunAnalyzer(ctx, req) +} diff --git a/internal/mocks/mock_proxy.go b/internal/mocks/mock_proxy.go index 6025dd3cc9..b58e6dcdb5 100644 --- a/internal/mocks/mock_proxy.go +++ b/internal/mocks/mock_proxy.go @@ -5772,6 +5772,65 @@ func (_c *MockProxy_RestoreRBAC_Call) RunAndReturn(run func(context.Context, *mi return _c } +// RunAnalyzer provides a mock function with given fields: _a0, _a1 +func (_m *MockProxy) RunAnalyzer(_a0 context.Context, _a1 *milvuspb.RunAnalyzerRequset) (*milvuspb.RunAnalyzerResponse, error) { + ret := _m.Called(_a0, _a1) + + if len(ret) == 0 { + panic("no return value specified for RunAnalyzer") + } + + var r0 *milvuspb.RunAnalyzerResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.RunAnalyzerRequset) (*milvuspb.RunAnalyzerResponse, error)); ok { + return rf(_a0, _a1) + } + if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.RunAnalyzerRequset) *milvuspb.RunAnalyzerResponse); ok { + r0 = rf(_a0, _a1) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*milvuspb.RunAnalyzerResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, *milvuspb.RunAnalyzerRequset) error); ok { + r1 = rf(_a0, _a1) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockProxy_RunAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RunAnalyzer' +type MockProxy_RunAnalyzer_Call struct { + *mock.Call +} + +// RunAnalyzer is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 *milvuspb.RunAnalyzerRequset +func (_e *MockProxy_Expecter) RunAnalyzer(_a0 interface{}, _a1 interface{}) *MockProxy_RunAnalyzer_Call { + return &MockProxy_RunAnalyzer_Call{Call: _e.mock.On("RunAnalyzer", _a0, _a1)} +} + +func (_c *MockProxy_RunAnalyzer_Call) Run(run func(_a0 context.Context, _a1 *milvuspb.RunAnalyzerRequset)) *MockProxy_RunAnalyzer_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(*milvuspb.RunAnalyzerRequset)) + }) + return _c +} + +func (_c *MockProxy_RunAnalyzer_Call) Return(_a0 *milvuspb.RunAnalyzerResponse, _a1 error) *MockProxy_RunAnalyzer_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockProxy_RunAnalyzer_Call) RunAndReturn(run func(context.Context, *milvuspb.RunAnalyzerRequset) (*milvuspb.RunAnalyzerResponse, error)) *MockProxy_RunAnalyzer_Call { + _c.Call.Return(run) + return _c +} + // Search provides a mock function with given fields: _a0, _a1 func (_m *MockProxy) Search(_a0 context.Context, _a1 *milvuspb.SearchRequest) (*milvuspb.SearchResults, error) { ret := _m.Called(_a0, _a1) diff --git a/internal/proxy/impl.go b/internal/proxy/impl.go index 7c2324c353..cf3f95c6dc 100644 --- a/internal/proxy/impl.go +++ b/internal/proxy/impl.go @@ -44,6 +44,7 @@ import ( "github.com/milvus-io/milvus/internal/http" "github.com/milvus-io/milvus/internal/proxy/connection" "github.com/milvus-io/milvus/internal/types" + "github.com/milvus-io/milvus/internal/util/ctokenizer" "github.com/milvus-io/milvus/internal/util/hookutil" "github.com/milvus-io/milvus/internal/util/importutilv2" "github.com/milvus-io/milvus/internal/util/streamingutil" @@ -6928,3 +6929,33 @@ func (node *Proxy) OperatePrivilegeGroup(ctx context.Context, req *milvuspb.Oper } return result, nil } + +func (node *Proxy) RunAnalyzer(ctx context.Context, req *milvuspb.RunAnalyzerRequset) (*milvuspb.RunAnalyzerResponse, error) { + // TODO: use collection analyzer when collection name and field name not none + tokenizer, err := ctokenizer.NewTokenizer(req.GetAnalyzerParams()) + if err != nil { + return &milvuspb.RunAnalyzerResponse{ + Status: merr.Status(err), + }, nil + } + defer tokenizer.Destroy() + + results := make([]*milvuspb.AnalyzerResult, len(req.GetPlaceholder())) + for i, text := range req.GetPlaceholder() { + stream := tokenizer.NewTokenStream(string(text)) + defer stream.Destroy() + tokens := []string{} + for stream.Advance() { + token := stream.Token() + tokens = append(tokens, token) + } + results[i] = &milvuspb.AnalyzerResult{ + Tokens: tokens, + } + } + + return &milvuspb.RunAnalyzerResponse{ + Status: merr.Status(nil), + Results: results, + }, nil +} diff --git a/internal/proxy/impl_test.go b/internal/proxy/impl_test.go index cdb47be484..7f55da7cfa 100644 --- a/internal/proxy/impl_test.go +++ b/internal/proxy/impl_test.go @@ -29,6 +29,7 @@ import ( "github.com/gin-gonic/gin" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/metadata" @@ -2194,3 +2195,22 @@ func TestAlterCollectionReplicateProperty(t *testing.T) { assert.NoError(t, err) assert.True(t, merr.Ok(statusResp)) } + +func TestRunAnalyzer(t *testing.T) { + p := &Proxy{} + // run analyzer with default params + resp, err := p.RunAnalyzer(context.Background(), &milvuspb.RunAnalyzerRequset{ + Placeholder: [][]byte{[]byte("test doc")}, + }) + require.NoError(t, err) + require.NoError(t, merr.Error(resp.GetStatus())) + assert.Equal(t, len(resp.GetResults()[0].GetTokens()), 2) + + // run analyzer with invalid params + resp, err = p.RunAnalyzer(context.Background(), &milvuspb.RunAnalyzerRequset{ + Placeholder: [][]byte{[]byte("test doc")}, + AnalyzerParams: "invalid json", + }) + require.NoError(t, err) + require.Error(t, merr.Error(resp.GetStatus())) +} diff --git a/pkg/go.mod b/pkg/go.mod index 9c757c9659..d4df256f57 100644 --- a/pkg/go.mod +++ b/pkg/go.mod @@ -15,7 +15,7 @@ require ( github.com/jolestar/go-commons-pool/v2 v2.1.2 github.com/json-iterator/go v1.1.12 github.com/klauspost/compress v1.17.7 - github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250107132849-4099923438e6 + github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250208062437-5af22aa4b559 github.com/nats-io/nats-server/v2 v2.10.12 github.com/nats-io/nats.go v1.34.1 github.com/panjf2000/ants/v2 v2.7.2 diff --git a/pkg/go.sum b/pkg/go.sum index 1bcc6630b2..47d8e4a323 100644 --- a/pkg/go.sum +++ b/pkg/go.sum @@ -492,8 +492,8 @@ github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119 h1:9VXijWu github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg= github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8= github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4= -github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250107132849-4099923438e6 h1:IUHMoxffuI2xbtMJJE/XK7kKzavZnN5sMiheIKTDVt8= -github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250107132849-4099923438e6/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs= +github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250208062437-5af22aa4b559 h1:c8n10eBkYU/HYaDUNAaKog4aIA3ZHO+GL7bHN2Ug/MA= +github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250208062437-5af22aa4b559/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs= github.com/milvus-io/pulsar-client-go v0.12.1 h1:O2JZp1tsYiO7C0MQ4hrUY/aJXnn2Gry6hpm7UodghmE= github.com/milvus-io/pulsar-client-go v0.12.1/go.mod h1:dkutuH4oS2pXiGm+Ti7fQZ4MRjrMPZ8IJeEGAWMeckk= github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=