admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-06 17:18:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: hybridsearch should support offset param in restful api (#43586)

Browse Source 
Add support of offset param for reqeustful. api and refine some constant
usage related #43556

Signed-off-by: xiaofanluan <xiaofan.luan@zilliz.com>
This commit is contained in:
Xiaofan 2025-07-28 22:15:36 +08:00 committed by GitHub
parent a29b3272b0
commit bd31b32167
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
19 changed files with 51 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/tools/migration/meta/210_to_220.go
View File

@ -161,7 +161,7 @@ func combineToCollectionIndexesMeta220(fieldIndexes FieldIndexes210, collectionI
}
newIndexParamsMap := make(map[string]string)
for _, kv := range indexInfo.IndexParams {
if kv.Key == common.IndexParamsKey {
if kv.Key == common.ParamsKey {
params, err := funcutil.JSONToMap(kv.Value)
if err != nil {
return nil, err

8
internal/distributed/proxy/httpserver/constant.go
View File

@ -17,6 +17,7 @@
package httpserver
import (
"github.com/milvus-io/milvus/internal/proxy"
"github.com/milvus-io/milvus/pkg/v2/util/metric"
)
@ -186,11 +187,8 @@ const (
)
const (
ParamAnnsField = "anns_field"
Params = "params"
ParamRoundDecimal = "round_decimal"
ParamOffset = "offset"
ParamLimit = "limit"
Params = proxy.ParamsKey
ParamRoundDecimal = proxy.RoundDecimalKey
ParamRadius = "radius"
ParamRangeFilter = "range_filter"
ParamGroupByField = "group_by_field"

6
internal/distributed/proxy/httpserver/handler_v1.go
View File

@ -525,10 +525,10 @@ func (h *HandlersV1) query(c *gin.Context) {
}
c.Set(ContextRequest, req)
if httpReq.Offset > 0 {
req.QueryParams = append(req.QueryParams, &commonpb.KeyValuePair{Key: ParamOffset, Value: strconv.FormatInt(int64(httpReq.Offset), 10)})
req.QueryParams = append(req.QueryParams, &commonpb.KeyValuePair{Key: proxy.OffsetKey, Value: strconv.FormatInt(int64(httpReq.Offset), 10)})
}
if httpReq.Limit > 0 {
req.QueryParams = append(req.QueryParams, &commonpb.KeyValuePair{Key: ParamLimit, Value: strconv.FormatInt(int64(httpReq.Limit), 10)})
req.QueryParams = append(req.QueryParams, &commonpb.KeyValuePair{Key: proxy.LimitKey, Value: strconv.FormatInt(int64(httpReq.Limit), 10)})
}
username, _ := c.Get(ContextUsername)
ctx := proxy.NewContextWithMetadata(c, username.(string), req.DbName)
@ -977,7 +977,7 @@ func (h *HandlersV1) search(c *gin.Context) {
{Key: common.TopKKey, Value: strconv.FormatInt(int64(httpReq.Limit), 10)},
{Key: Params, Value: string(bs)},
{Key: ParamRoundDecimal, Value: "-1"},
{Key: ParamOffset, Value: strconv.FormatInt(int64(httpReq.Offset), 10)},
{Key: proxy.OffsetKey, Value: strconv.FormatInt(int64(httpReq.Offset), 10)},
}
username, _ := c.Get(ContextUsername)

15
internal/distributed/proxy/httpserver/handler_v2.go
View File

@ -837,10 +837,10 @@ func (h *HandlersV2) query(ctx context.Context, c *gin.Context, anyReq any, dbNa
req.ExprTemplateValues = generateExpressionTemplate(httpReq.ExprParams)
c.Set(ContextRequest, req)
if httpReq.Offset > 0 {
req.QueryParams = append(req.QueryParams, &commonpb.KeyValuePair{Key: ParamOffset, Value: strconv.FormatInt(int64(httpReq.Offset), 10)})
req.QueryParams = append(req.QueryParams, &commonpb.KeyValuePair{Key: proxy.OffsetKey, Value: strconv.FormatInt(int64(httpReq.Offset), 10)})
}
if httpReq.Limit > 0 && !matchCountRule(httpReq.OutputFields) {
req.QueryParams = append(req.QueryParams, &commonpb.KeyValuePair{Key: ParamLimit, Value: strconv.FormatInt(int64(httpReq.Limit), 10)})
req.QueryParams = append(req.QueryParams, &commonpb.KeyValuePair{Key: proxy.LimitKey, Value: strconv.FormatInt(int64(httpReq.Limit), 10)})
}
resp, err := wrapperProxyWithLimit(ctx, c, req, h.checkAuth, false, "/milvus.proto.milvus.MilvusService/Query", true, h.proxy, func(reqCtx context.Context, req any) (interface{}, error) {
return h.proxy.Query(reqCtx, req.(*milvuspb.QueryRequest))
@ -1201,7 +1201,7 @@ func (h *HandlersV2) search(ctx context.Context, c *gin.Context, anyReq any, dbN
return nil, err
}
searchParams = append(searchParams, &commonpb.KeyValuePair{Key: common.TopKKey, Value: strconv.FormatInt(int64(httpReq.Limit), 10)})
searchParams = append(searchParams, &commonpb.KeyValuePair{Key: ParamOffset, Value: strconv.FormatInt(int64(httpReq.Offset), 10)})
searchParams = append(searchParams, &commonpb.KeyValuePair{Key: proxy.OffsetKey, Value: strconv.FormatInt(int64(httpReq.Offset), 10)})
if httpReq.GroupByField != "" {
searchParams = append(searchParams, &commonpb.KeyValuePair{Key: ParamGroupByField, Value: httpReq.GroupByField})
}
@ -1297,7 +1297,7 @@ func (h *HandlersV2) advancedSearch(ctx context.Context, c *gin.Context, anyReq
return nil, err
}
searchParams = append(searchParams, &commonpb.KeyValuePair{Key: common.TopKKey, Value: strconv.FormatInt(int64(subReq.Limit), 10)})
searchParams = append(searchParams, &commonpb.KeyValuePair{Key: ParamOffset, Value: strconv.FormatInt(int64(subReq.Offset), 10)})
searchParams = append(searchParams, &commonpb.KeyValuePair{Key: proxy.OffsetKey, Value: strconv.FormatInt(int64(subReq.Offset), 10)})
searchParams = append(searchParams, &commonpb.KeyValuePair{Key: proxy.AnnsFieldKey, Value: subReq.AnnsField})
placeholderGroup, err := generatePlaceholderGroup(ctx, searchArray[i].Raw, collSchema, subReq.AnnsField)
if err != nil {
@ -1321,11 +1321,14 @@ func (h *HandlersV2) advancedSearch(ctx context.Context, c *gin.Context, anyReq
searchReq.ExprTemplateValues = generateExpressionTemplate(subReq.ExprParams)
req.Requests = append(req.Requests, searchReq)
}
bs, _ := json.Marshal(httpReq.Rerank.Params)
// leave the rerank check to proxy side
req.RankParams = []*commonpb.KeyValuePair{
{Key: proxy.RankTypeKey, Value: httpReq.Rerank.Strategy},
{Key: proxy.RankParamsKey, Value: string(bs)},
{Key: ParamLimit, Value: strconv.FormatInt(int64(httpReq.Limit), 10)},
{Key: proxy.ParamsKey, Value: string(bs)},
{Key: proxy.LimitKey, Value: strconv.FormatInt(int64(httpReq.Limit), 10)},
{Key: proxy.OffsetKey, Value: strconv.FormatInt(int64(httpReq.Offset), 10)},
{Key: ParamRoundDecimal, Value: "-1"},
}
if httpReq.GroupByField != "" {

2
internal/distributed/proxy/httpserver/handler_v2_test.go
View File

@ -2141,7 +2141,7 @@ func TestDML(t *testing.T) {
mp.EXPECT().Query(mock.Anything, mock.Anything).RunAndReturn(func(ctx context.Context, req *milvuspb.QueryRequest) (*milvuspb.QueryResults, error) {
if matchCountRule(req.OutputFields) {
for _, pair := range req.QueryParams {
if pair.GetKey() == ParamLimit {
if pair.GetKey() == proxy.LimitKey {
return nil, errors.New("mock error")
}
}

1
internal/distributed/proxy/httpserver/request_v2.go
View File

@ -323,6 +323,7 @@ type HybridSearchReq struct {
Search []SubSearchReq `json:"search"`
Rerank Rand `json:"rerank"`
Limit int32 `json:"limit"`
Offset int32 `json:"offset"`
GroupByField string `json:"groupingField"`
GroupSize int32 `json:"groupSize"`
StrictGroupSize bool `json:"strictGroupSize"`

2
internal/distributed/proxy/httpserver/utils.go
View File

@ -1663,7 +1663,7 @@ func convertToExtraParams(indexParam IndexParam) ([]*commonpb.KeyValuePair, erro
if err != nil {
return nil, err
}
params = append(params, &commonpb.KeyValuePair{Key: common.IndexParamsKey, Value: string(v)})
params = append(params, &commonpb.KeyValuePair{Key: common.ParamsKey, Value: string(v)})
}
return params, nil
}

2
internal/distributed/proxy/httpserver/utils_test.go
View File

@ -2297,7 +2297,7 @@ func TestConvertToExtraParams(t *testing.T) {
if pair.Key == common.IndexTypeKey {
assert.Equal(t, "IVF_FLAT", pair.Value)
}
if pair.Key == common.IndexParamsKey {
if pair.Key == common.ParamsKey {
assert.Equal(t, string("{\"nlist\":128}"), pair.Value)
}
}

8
internal/proxy/proxy_test.go
View File

@ -1525,7 +1525,7 @@ func TestProxy(t *testing.T) {
assert.NoError(t, err)
searchParams := []*commonpb.KeyValuePair{
{Key: MetricTypeKey, Value: metric.L2},
{Key: SearchParamsKey, Value: string(b)},
{Key: ParamsKey, Value: string(b)},
{Key: AnnsFieldKey, Value: floatVecField},
{Key: TopKKey, Value: strconv.Itoa(topk)},
{Key: RoundDecimalKey, Value: strconv.Itoa(roundDecimal)},
@ -1558,7 +1558,7 @@ func TestProxy(t *testing.T) {
assert.NoError(t, err)
searchParams := []*commonpb.KeyValuePair{
{Key: MetricTypeKey, Value: metric.L2},
{Key: SearchParamsKey, Value: string(b)},
{Key: ParamsKey, Value: string(b)},
{Key: AnnsFieldKey, Value: floatVecField},
{Key: TopKKey, Value: strconv.Itoa(topk)},
{Key: RoundDecimalKey, Value: strconv.Itoa(roundDecimal)},
@ -1597,7 +1597,7 @@ func TestProxy(t *testing.T) {
assert.NoError(t, err)
rankParams := []*commonpb.KeyValuePair{
{Key: RankTypeKey, Value: "rrf"},
{Key: RankParamsKey, Value: string(b)},
{Key: ParamsKey, Value: string(b)},
{Key: LimitKey, Value: strconv.Itoa(topk)},
{Key: RoundDecimalKey, Value: strconv.Itoa(roundDecimal)},
}
@ -1655,7 +1655,7 @@ func TestProxy(t *testing.T) {
assert.NoError(t, err)
searchParams := []*commonpb.KeyValuePair{
{Key: MetricTypeKey, Value: metric.L2},
{Key: SearchParamsKey, Value: string(b)},
{Key: ParamsKey, Value: string(b)},
{Key: AnnsFieldKey, Value: floatVecField},
{Key: TopKKey, Value: strconv.Itoa(topk)},
{Key: RoundDecimalKey, Value: strconv.Itoa(roundDecimal)},

2
internal/proxy/search_util.go
View File

@ -248,7 +248,7 @@ func parseSearchInfo(searchParamsPair []*commonpb.KeyValuePair, schema *schemapb
}
// 4. parse search param str
searchParamStr, err := funcutil.GetAttrByKeyFromRepeatedKV(SearchParamsKey, searchParamsPair)
searchParamStr, err := funcutil.GetAttrByKeyFromRepeatedKV(ParamsKey, searchParamsPair)
if err != nil {
searchParamStr = ""
}

3
internal/proxy/task.go
View File

@ -63,7 +63,7 @@ const (
TopKKey = "topk"
NQKey = "nq"
MetricTypeKey = common.MetricTypeKey
SearchParamsKey = "params"
ParamsKey = common.ParamsKey
ExprParamsKey = "expr_params"
RoundDecimalKey = "round_decimal"
OffsetKey = "offset"
@ -119,7 +119,6 @@ const (
minFloat32 = -1 * float32(math.MaxFloat32)
RankTypeKey = "strategy"
RankParamsKey = "params"
RRFParamsKey = "k"
WeightsParamsKey = "weights"
NormScoreKey = "norm_score"

2
internal/proxy/task_index.go
View File

@ -179,7 +179,7 @@ func (cit *createIndexTask) parseIndexParams(ctx context.Context) error {
return merr.WrapErrParameterInvalidMsg("duplicated index param (key=%s) (value=%s) found", kv.GetKey(), kv.GetValue())
}
keys.Insert(kv.GetKey())
if kv.Key == common.IndexParamsKey {
if kv.Key == common.ParamsKey {
params, err := funcutil.JSONToMap(kv.Value)
if err != nil {
return err

22
internal/proxy/task_index_test.go
View File

@ -312,7 +312,7 @@ func Test_sparse_parseIndexParams(t *testing.T) {
Value: "IP",
},
{
Key: common.IndexParamsKey,
Key: common.ParamsKey,
Value: "{\"drop_ratio_build\": 0.3}",
},
},
@ -382,7 +382,7 @@ func Test_parseIndexParams(t *testing.T) {
Value: "IP",
},
{
Key: common.IndexParamsKey,
Key: common.ParamsKey,
Value: "{\"M\": 48, \"efConstruction\": 64}",
},
{
@ -467,7 +467,7 @@ func Test_parseIndexParams(t *testing.T) {
Value: "L2",
},
{
Key: common.IndexParamsKey,
Key: common.ParamsKey,
Value: "{\"nlist\": 100}",
},
{
@ -564,7 +564,7 @@ func Test_parseIndexParams(t *testing.T) {
Value: "IP",
},
{
Key: common.IndexParamsKey,
Key: common.ParamsKey,
Value: "{\"M\": 48, \"efConstruction\": 64}",
},
{
@ -803,7 +803,7 @@ func Test_parseIndexParams(t *testing.T) {
Value: "IP",
},
{
Key: common.IndexParamsKey,
Key: common.ParamsKey,
Value: "{\"M\": 48, \"efConstruction\": 64}",
},
{
@ -848,7 +848,7 @@ func Test_parseIndexParams(t *testing.T) {
Value: "IP",
},
{
Key: common.IndexParamsKey,
Key: common.ParamsKey,
Value: "{\"M\": 48, \"efConstruction\": 64}",
},
{
@ -1168,7 +1168,7 @@ func Test_ngram_parseIndexParams(t *testing.T) {
req: &milvuspb.CreateIndexRequest{
ExtraParams: []*commonpb.KeyValuePair{
{Key: common.IndexTypeKey, Value: "NGRAM"},
{Key: common.IndexParamsKey, Value: "{\"min_gram\": \"2\", \"max_gram\": \"3\"}"},
{Key: common.ParamsKey, Value: "{\"min_gram\": \"2\", \"max_gram\": \"3\"}"},
},
},
fieldSchema: &schemapb.FieldSchema{
@ -1190,7 +1190,7 @@ func Test_ngram_parseIndexParams(t *testing.T) {
req: &milvuspb.CreateIndexRequest{
ExtraParams: []*commonpb.KeyValuePair{
{Key: common.IndexTypeKey, Value: "NGRAM"},
{Key: common.IndexParamsKey, Value: "{\"min_gram\": \"2\", \"max_gram\": \"3\"}"},
{Key: common.ParamsKey, Value: "{\"min_gram\": \"2\", \"max_gram\": \"3\"}"},
},
},
fieldSchema: &schemapb.FieldSchema{
@ -1206,7 +1206,7 @@ func Test_ngram_parseIndexParams(t *testing.T) {
req: &milvuspb.CreateIndexRequest{
ExtraParams: []*commonpb.KeyValuePair{
{Key: common.IndexTypeKey, Value: "NGRAM"},
{Key: common.IndexParamsKey, Value: "{\"min_gram\": \"2\"}"},
{Key: common.ParamsKey, Value: "{\"min_gram\": \"2\"}"},
},
},
fieldSchema: &schemapb.FieldSchema{
@ -1222,7 +1222,7 @@ func Test_ngram_parseIndexParams(t *testing.T) {
req: &milvuspb.CreateIndexRequest{
ExtraParams: []*commonpb.KeyValuePair{
{Key: common.IndexTypeKey, Value: "NGRAM"},
{Key: common.IndexParamsKey, Value: "{\"min_gram\": \"a\", \"max_gram\": \"3\"}"},
{Key: common.ParamsKey, Value: "{\"min_gram\": \"a\", \"max_gram\": \"3\"}"},
},
},
fieldSchema: &schemapb.FieldSchema{
@ -1238,7 +1238,7 @@ func Test_ngram_parseIndexParams(t *testing.T) {
req: &milvuspb.CreateIndexRequest{
ExtraParams: []*commonpb.KeyValuePair{
{Key: common.IndexTypeKey, Value: "NGRAM"},
{Key: common.IndexParamsKey, Value: "{\"min_gram\": \"5\", \"max_gram\": \"3\"}"},
{Key: common.ParamsKey, Value: "{\"min_gram\": \"5\", \"max_gram\": \"3\"}"},
},
},
fieldSchema: &schemapb.FieldSchema{

8
internal/proxy/task_search_test.go
View File

@ -597,7 +597,7 @@ func getValidSearchParams() []*commonpb.KeyValuePair {
Value: metric.L2,
},
{
Key: SearchParamsKey,
Key: ParamsKey,
Value: `{"nprobe": 10}`,
},
{
@ -3080,7 +3080,7 @@ func TestTaskSearch_parseSearchInfo(t *testing.T) {
noMetricTypeParams := getBaseSearchParams()
noMetricTypeParams = append(noMetricTypeParams, &commonpb.KeyValuePair{
Key: SearchParamsKey,
Key: ParamsKey,
Value: `{"nprobe": 10}`,
})
@ -3226,7 +3226,7 @@ func TestTaskSearch_parseSearchInfo(t *testing.T) {
// no roundDecimal is valid
noRoundDecimal := append(spNoSearchParams, &commonpb.KeyValuePair{
Key: SearchParamsKey,
Key: ParamsKey,
Value: `{"nprobe": 10}`,
})
@ -3304,7 +3304,7 @@ func TestTaskSearch_parseSearchInfo(t *testing.T) {
})
t.Run("check range-search and groupBy", func(t *testing.T) {
normalParam := getValidSearchParams()
resetSearchParamsValue(normalParam, SearchParamsKey, `{"nprobe": 10, "radius":0.2}`)
resetSearchParamsValue(normalParam, ParamsKey, `{"nprobe": 10, "radius":0.2}`)
normalParam = append(normalParam, &commonpb.KeyValuePair{
Key: GroupByFieldKey,
Value: "string_field",

2
internal/proxy/task_test.go
View File

@ -490,7 +490,7 @@ func constructSearchRequest(
Value: metric.L2,
},
{
Key: SearchParamsKey,
Key: ParamsKey,
Value: string(b),
},
{

2
pkg/common/common.go
View File

@ -135,7 +135,7 @@ const (
CollectionKey = "collection"
RecallEvalKey = "recall_eval"
IndexParamsKey = "params"
ParamsKey = "params"
IndexTypeKey = "index_type"
MetricTypeKey = "metric_type"
DimKey = "dim"

6
pkg/util/constant.go
View File

@ -23,7 +23,6 @@ import (
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
"github.com/milvus-io/milvus/pkg/v2/common"
"github.com/milvus-io/milvus/pkg/v2/util/typeutil"
)
@ -81,11 +80,6 @@ const (
GB = 1024 * 1024 * 1024
)
const (
// ParamsKeyToParse is the key of the param to build index.
ParamsKeyToParse = common.IndexParamsKey
)
var (
DefaultRoles = []string{RoleAdmin, RolePublic}
BuiltinRoles = []string{}

8
tests/integration/hellomilvus/hybridsearch_test.go
View File

@ -206,7 +206,7 @@ func (s *HelloMilvusSuite) TestHybridSearch() {
s.NoError(err)
hSearchReq.RankParams = []*commonpb.KeyValuePair{
{Key: proxy.RankTypeKey, Value: "rrf"},
{Key: proxy.RankParamsKey, Value: string(b)},
{Key: proxy.ParamsKey, Value: string(b)},
{Key: proxy.LimitKey, Value: strconv.Itoa(topk)},
{Key: proxy.RoundDecimalKey, Value: strconv.Itoa(roundDecimal)},
}
@ -232,7 +232,7 @@ func (s *HelloMilvusSuite) TestHybridSearch() {
}
hSearchReq.RankParams = []*commonpb.KeyValuePair{
{Key: proxy.RankTypeKey, Value: "weighted"},
{Key: proxy.RankParamsKey, Value: string(b)},
{Key: proxy.ParamsKey, Value: string(b)},
{Key: proxy.LimitKey, Value: strconv.Itoa(topk)},
}
@ -371,7 +371,7 @@ func (s *HelloMilvusSuite) TestHybridSearchSingleSubReq() {
s.NoError(err)
hSearchReq.RankParams = []*commonpb.KeyValuePair{
{Key: proxy.RankTypeKey, Value: "rrf"},
{Key: proxy.RankParamsKey, Value: string(b)},
{Key: proxy.ParamsKey, Value: string(b)},
{Key: proxy.LimitKey, Value: strconv.Itoa(topk)},
{Key: proxy.RoundDecimalKey, Value: strconv.Itoa(roundDecimal)},
}
@ -397,7 +397,7 @@ func (s *HelloMilvusSuite) TestHybridSearchSingleSubReq() {
}
hSearchReq.RankParams = []*commonpb.KeyValuePair{
{Key: proxy.RankTypeKey, Value: "weighted"},
{Key: proxy.RankParamsKey, Value: string(b)},
{Key: proxy.ParamsKey, Value: string(b)},
{Key: proxy.LimitKey, Value: strconv.Itoa(topk)},
}

6
tests/integration/util_query.go
View File

@ -30,6 +30,7 @@ import (
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/internal/proxy"
"github.com/milvus-io/milvus/pkg/v2/common"
"github.com/milvus-io/milvus/pkg/v2/util/merr"
"github.com/milvus-io/milvus/pkg/v2/util/metricsinfo"
@ -42,7 +43,6 @@ const (
TopKKey = "topk"
NQKey = "nq"
MetricTypeKey = common.MetricTypeKey
SearchParamsKey = common.IndexParamsKey
RoundDecimalKey = "round_decimal"
OffsetKey = "offset"
LimitKey = "limit"
@ -199,7 +199,7 @@ func ConstructSearchRequest(
Value: metricType,
},
{
Key: SearchParamsKey,
Key: proxy.ParamsKey,
Value: string(b),
},
{
@ -258,7 +258,7 @@ func ConstructSearchRequestWithConsistencyLevel(
Value: metricType,
},
{
Key: SearchParamsKey,
Key: proxy.ParamsKey,
Value: string(b),
},
{