mirror of
https://gitee.com/milvus-io/milvus.git
synced 2025-12-06 17:18:35 +08:00
enhance: support run analyzer (#39723)
relate: https://github.com/milvus-io/milvus/issues/39705 --------- Signed-off-by: aoiasd <zhicheng.yue@zilliz.com>
This commit is contained in:
aoiasd
GitHub
parent
09b9131321
commit
5e6f6af65d
2
go.mod
2
go.mod
@ -23,7 +23,7 @@ require (
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
|
||||
github.com/klauspost/compress v1.17.9
|
||||
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250107132849-4099923438e6
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250208062437-5af22aa4b559
|
||||
github.com/minio/minio-go/v7 v7.0.73
|
||||
github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81
|
||||
github.com/prometheus/client_golang v1.14.0
|
||||
|
||||
4
go.sum
4
go.sum
@ -660,8 +660,8 @@ github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119 h1:9VXijWu
|
||||
github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
|
||||
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
|
||||
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250107132849-4099923438e6 h1:IUHMoxffuI2xbtMJJE/XK7kKzavZnN5sMiheIKTDVt8=
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250107132849-4099923438e6/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250208062437-5af22aa4b559 h1:c8n10eBkYU/HYaDUNAaKog4aIA3ZHO+GL7bHN2Ug/MA=
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250208062437-5af22aa4b559/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
|
||||
github.com/milvus-io/pulsar-client-go v0.12.1 h1:O2JZp1tsYiO7C0MQ4hrUY/aJXnn2Gry6hpm7UodghmE=
|
||||
github.com/milvus-io/pulsar-client-go v0.12.1/go.mod h1:dkutuH4oS2pXiGm+Ti7fQZ4MRjrMPZ8IJeEGAWMeckk=
|
||||
github.com/minio/asm2plan9s v0.0.0-20200509001527-cdd76441f9d8 h1:AMFGa4R4MiIpspGNG7Z948v4n35fFGB3RR3G/ry4FWs=
|
||||
|
||||
@ -1157,3 +1157,7 @@ func (s *Server) InvalidateShardLeaderCache(ctx context.Context, req *proxypb.In
|
||||
func (s *Server) DescribeDatabase(ctx context.Context, req *milvuspb.DescribeDatabaseRequest) (*milvuspb.DescribeDatabaseResponse, error) {
|
||||
return s.proxy.DescribeDatabase(ctx, req)
|
||||
}
|
||||
|
||||
func (s *Server) RunAnalyzer(ctx context.Context, req *milvuspb.RunAnalyzerRequset) (*milvuspb.RunAnalyzerResponse, error) {
|
||||
return s.proxy.RunAnalyzer(ctx, req)
|
||||
}
|
||||
|
||||
@ -5772,6 +5772,65 @@ func (_c *MockProxy_RestoreRBAC_Call) RunAndReturn(run func(context.Context, *mi
|
||||
return _c
|
||||
}
|
||||
|
||||
// RunAnalyzer provides a mock function with given fields: _a0, _a1
|
||||
func (_m *MockProxy) RunAnalyzer(_a0 context.Context, _a1 *milvuspb.RunAnalyzerRequset) (*milvuspb.RunAnalyzerResponse, error) {
|
||||
ret := _m.Called(_a0, _a1)
|
||||
|
||||
if len(ret) == 0 {
|
||||
panic("no return value specified for RunAnalyzer")
|
||||
}
|
||||
|
||||
var r0 *milvuspb.RunAnalyzerResponse
|
||||
var r1 error
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.RunAnalyzerRequset) (*milvuspb.RunAnalyzerResponse, error)); ok {
|
||||
return rf(_a0, _a1)
|
||||
}
|
||||
if rf, ok := ret.Get(0).(func(context.Context, *milvuspb.RunAnalyzerRequset) *milvuspb.RunAnalyzerResponse); ok {
|
||||
r0 = rf(_a0, _a1)
|
||||
} else {
|
||||
if ret.Get(0) != nil {
|
||||
r0 = ret.Get(0).(*milvuspb.RunAnalyzerResponse)
|
||||
}
|
||||
}
|
||||
|
||||
if rf, ok := ret.Get(1).(func(context.Context, *milvuspb.RunAnalyzerRequset) error); ok {
|
||||
r1 = rf(_a0, _a1)
|
||||
} else {
|
||||
r1 = ret.Error(1)
|
||||
}
|
||||
|
||||
return r0, r1
|
||||
}
|
||||
|
||||
// MockProxy_RunAnalyzer_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'RunAnalyzer'
|
||||
type MockProxy_RunAnalyzer_Call struct {
|
||||
*mock.Call
|
||||
}
|
||||
|
||||
// RunAnalyzer is a helper method to define mock.On call
|
||||
// - _a0 context.Context
|
||||
// - _a1 *milvuspb.RunAnalyzerRequset
|
||||
func (_e *MockProxy_Expecter) RunAnalyzer(_a0 interface{}, _a1 interface{}) *MockProxy_RunAnalyzer_Call {
|
||||
return &MockProxy_RunAnalyzer_Call{Call: _e.mock.On("RunAnalyzer", _a0, _a1)}
|
||||
}
|
||||
|
||||
func (_c *MockProxy_RunAnalyzer_Call) Run(run func(_a0 context.Context, _a1 *milvuspb.RunAnalyzerRequset)) *MockProxy_RunAnalyzer_Call {
|
||||
_c.Call.Run(func(args mock.Arguments) {
|
||||
run(args[0].(context.Context), args[1].(*milvuspb.RunAnalyzerRequset))
|
||||
})
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockProxy_RunAnalyzer_Call) Return(_a0 *milvuspb.RunAnalyzerResponse, _a1 error) *MockProxy_RunAnalyzer_Call {
|
||||
_c.Call.Return(_a0, _a1)
|
||||
return _c
|
||||
}
|
||||
|
||||
func (_c *MockProxy_RunAnalyzer_Call) RunAndReturn(run func(context.Context, *milvuspb.RunAnalyzerRequset) (*milvuspb.RunAnalyzerResponse, error)) *MockProxy_RunAnalyzer_Call {
|
||||
_c.Call.Return(run)
|
||||
return _c
|
||||
}
|
||||
|
||||
// Search provides a mock function with given fields: _a0, _a1
|
||||
func (_m *MockProxy) Search(_a0 context.Context, _a1 *milvuspb.SearchRequest) (*milvuspb.SearchResults, error) {
|
||||
ret := _m.Called(_a0, _a1)
|
||||
|
||||
@ -44,6 +44,7 @@ import (
|
||||
"github.com/milvus-io/milvus/internal/http"
|
||||
"github.com/milvus-io/milvus/internal/proxy/connection"
|
||||
"github.com/milvus-io/milvus/internal/types"
|
||||
"github.com/milvus-io/milvus/internal/util/ctokenizer"
|
||||
"github.com/milvus-io/milvus/internal/util/hookutil"
|
||||
"github.com/milvus-io/milvus/internal/util/importutilv2"
|
||||
"github.com/milvus-io/milvus/internal/util/streamingutil"
|
||||
@ -6928,3 +6929,33 @@ func (node *Proxy) OperatePrivilegeGroup(ctx context.Context, req *milvuspb.Oper
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (node *Proxy) RunAnalyzer(ctx context.Context, req *milvuspb.RunAnalyzerRequset) (*milvuspb.RunAnalyzerResponse, error) {
|
||||
// TODO: use collection analyzer when collection name and field name not none
|
||||
tokenizer, err := ctokenizer.NewTokenizer(req.GetAnalyzerParams())
|
||||
if err != nil {
|
||||
return &milvuspb.RunAnalyzerResponse{
|
||||
Status: merr.Status(err),
|
||||
}, nil
|
||||
}
|
||||
defer tokenizer.Destroy()
|
||||
|
||||
results := make([]*milvuspb.AnalyzerResult, len(req.GetPlaceholder()))
|
||||
for i, text := range req.GetPlaceholder() {
|
||||
stream := tokenizer.NewTokenStream(string(text))
|
||||
defer stream.Destroy()
|
||||
tokens := []string{}
|
||||
for stream.Advance() {
|
||||
token := stream.Token()
|
||||
tokens = append(tokens, token)
|
||||
}
|
||||
results[i] = &milvuspb.AnalyzerResult{
|
||||
Tokens: tokens,
|
||||
}
|
||||
}
|
||||
|
||||
return &milvuspb.RunAnalyzerResponse{
|
||||
Status: merr.Status(nil),
|
||||
Results: results,
|
||||
}, nil
|
||||
}
|
||||
|
||||
@ -29,6 +29,7 @@ import (
|
||||
"github.com/gin-gonic/gin"
|
||||
"github.com/stretchr/testify/assert"
|
||||
"github.com/stretchr/testify/mock"
|
||||
"github.com/stretchr/testify/require"
|
||||
"go.uber.org/zap"
|
||||
"google.golang.org/grpc"
|
||||
"google.golang.org/grpc/metadata"
|
||||
@ -2194,3 +2195,22 @@ func TestAlterCollectionReplicateProperty(t *testing.T) {
|
||||
assert.NoError(t, err)
|
||||
assert.True(t, merr.Ok(statusResp))
|
||||
}
|
||||
|
||||
func TestRunAnalyzer(t *testing.T) {
|
||||
p := &Proxy{}
|
||||
// run analyzer with default params
|
||||
resp, err := p.RunAnalyzer(context.Background(), &milvuspb.RunAnalyzerRequset{
|
||||
Placeholder: [][]byte{[]byte("test doc")},
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.NoError(t, merr.Error(resp.GetStatus()))
|
||||
assert.Equal(t, len(resp.GetResults()[0].GetTokens()), 2)
|
||||
|
||||
// run analyzer with invalid params
|
||||
resp, err = p.RunAnalyzer(context.Background(), &milvuspb.RunAnalyzerRequset{
|
||||
Placeholder: [][]byte{[]byte("test doc")},
|
||||
AnalyzerParams: "invalid json",
|
||||
})
|
||||
require.NoError(t, err)
|
||||
require.Error(t, merr.Error(resp.GetStatus()))
|
||||
}
|
||||
|
||||
@ -15,7 +15,7 @@ require (
|
||||
github.com/jolestar/go-commons-pool/v2 v2.1.2
|
||||
github.com/json-iterator/go v1.1.12
|
||||
github.com/klauspost/compress v1.17.7
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250107132849-4099923438e6
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250208062437-5af22aa4b559
|
||||
github.com/nats-io/nats-server/v2 v2.10.12
|
||||
github.com/nats-io/nats.go v1.34.1
|
||||
github.com/panjf2000/ants/v2 v2.7.2
|
||||
|
||||
@ -492,8 +492,8 @@ github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119 h1:9VXijWu
|
||||
github.com/milvus-io/cgosymbolizer v0.0.0-20240722103217-b7dee0e50119/go.mod h1:DvXTE/K/RtHehxU8/GtDs4vFtfw64jJ3PaCnFri8CRg=
|
||||
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b h1:TfeY0NxYxZzUfIfYe5qYDBzt4ZYRqzUjTR6CvUzjat8=
|
||||
github.com/milvus-io/gorocksdb v0.0.0-20220624081344-8c5f4212846b/go.mod h1:iwW+9cWfIzzDseEBCCeDSN5SD16Tidvy8cwQ7ZY8Qj4=
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250107132849-4099923438e6 h1:IUHMoxffuI2xbtMJJE/XK7kKzavZnN5sMiheIKTDVt8=
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250107132849-4099923438e6/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250208062437-5af22aa4b559 h1:c8n10eBkYU/HYaDUNAaKog4aIA3ZHO+GL7bHN2Ug/MA=
|
||||
github.com/milvus-io/milvus-proto/go-api/v2 v2.5.0-beta.0.20250208062437-5af22aa4b559/go.mod h1:/6UT4zZl6awVeXLeE7UGDWZvXj3IWkRsh3mqsn0DiAs=
|
||||
github.com/milvus-io/pulsar-client-go v0.12.1 h1:O2JZp1tsYiO7C0MQ4hrUY/aJXnn2Gry6hpm7UodghmE=
|
||||
github.com/milvus-io/pulsar-client-go v0.12.1/go.mod h1:dkutuH4oS2pXiGm+Ti7fQZ4MRjrMPZ8IJeEGAWMeckk=
|
||||
github.com/minio/highwayhash v1.0.2 h1:Aak5U0nElisjDCfPSG79Tgzkn2gl66NxOMspRrKnA/g=
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user