diff --git a/internal/distributed/proxy/httpserver/utils.go b/internal/distributed/proxy/httpserver/utils.go index 262750ab4b..6b228b9f30 100644 --- a/internal/distributed/proxy/httpserver/utils.go +++ b/internal/distributed/proxy/httpserver/utils.go @@ -1315,46 +1315,46 @@ func buildQueryResp(rowsNum int64, needFields []string, fieldDataList []*schemap columnNum := len(fieldDataList) if rowsNum == int64(0) { // always if columnNum > 0 { - switch fieldDataList[0].Type { + switch fieldDataList[0].GetType() { case schemapb.DataType_Bool: - rowsNum = int64(len(fieldDataList[0].GetScalars().GetBoolData().Data)) + rowsNum = int64(len(fieldDataList[0].GetScalars().GetBoolData().GetData())) case schemapb.DataType_Int8: - rowsNum = int64(len(fieldDataList[0].GetScalars().GetIntData().Data)) + rowsNum = int64(len(fieldDataList[0].GetScalars().GetIntData().GetData())) case schemapb.DataType_Int16: - rowsNum = int64(len(fieldDataList[0].GetScalars().GetIntData().Data)) + rowsNum = int64(len(fieldDataList[0].GetScalars().GetIntData().GetData())) case schemapb.DataType_Int32: - rowsNum = int64(len(fieldDataList[0].GetScalars().GetIntData().Data)) + rowsNum = int64(len(fieldDataList[0].GetScalars().GetIntData().GetData())) case schemapb.DataType_Int64: - rowsNum = int64(len(fieldDataList[0].GetScalars().GetLongData().Data)) + rowsNum = int64(len(fieldDataList[0].GetScalars().GetLongData().GetData())) case schemapb.DataType_Float: - rowsNum = int64(len(fieldDataList[0].GetScalars().GetFloatData().Data)) + rowsNum = int64(len(fieldDataList[0].GetScalars().GetFloatData().GetData())) case schemapb.DataType_Double: - rowsNum = int64(len(fieldDataList[0].GetScalars().GetDoubleData().Data)) + rowsNum = int64(len(fieldDataList[0].GetScalars().GetDoubleData().GetData())) case schemapb.DataType_String: - rowsNum = int64(len(fieldDataList[0].GetScalars().GetStringData().Data)) + rowsNum = int64(len(fieldDataList[0].GetScalars().GetStringData().GetData())) case schemapb.DataType_VarChar: - rowsNum = int64(len(fieldDataList[0].GetScalars().GetStringData().Data)) + rowsNum = int64(len(fieldDataList[0].GetScalars().GetStringData().GetData())) case schemapb.DataType_Array: - rowsNum = int64(len(fieldDataList[0].GetScalars().GetArrayData().Data)) + rowsNum = int64(len(fieldDataList[0].GetScalars().GetArrayData().GetData())) case schemapb.DataType_JSON: - rowsNum = int64(len(fieldDataList[0].GetScalars().GetJsonData().Data)) + rowsNum = int64(len(fieldDataList[0].GetScalars().GetJsonData().GetData())) case schemapb.DataType_BinaryVector: rowsNum = int64(len(fieldDataList[0].GetVectors().GetBinaryVector())*8) / fieldDataList[0].GetVectors().GetDim() case schemapb.DataType_FloatVector: - rowsNum = int64(len(fieldDataList[0].GetVectors().GetFloatVector().Data)) / fieldDataList[0].GetVectors().GetDim() + rowsNum = int64(len(fieldDataList[0].GetVectors().GetFloatVector().GetData())) / fieldDataList[0].GetVectors().GetDim() case schemapb.DataType_Float16Vector: rowsNum = int64(len(fieldDataList[0].GetVectors().GetFloat16Vector())/2) / fieldDataList[0].GetVectors().GetDim() case schemapb.DataType_BFloat16Vector: rowsNum = int64(len(fieldDataList[0].GetVectors().GetBfloat16Vector())/2) / fieldDataList[0].GetVectors().GetDim() case schemapb.DataType_SparseFloatVector: - rowsNum = int64(len(fieldDataList[0].GetVectors().GetSparseFloatVector().Contents)) + rowsNum = int64(len(fieldDataList[0].GetVectors().GetSparseFloatVector().GetContents())) case schemapb.DataType_Int8Vector: rowsNum = int64(len(fieldDataList[0].GetVectors().GetInt8Vector())) / fieldDataList[0].GetVectors().GetDim() default: - return nil, fmt.Errorf("the type(%v) of field(%v) is not supported, use other sdk please", fieldDataList[0].Type, fieldDataList[0].FieldName) + return nil, fmt.Errorf("the type(%v) of field(%v) is not supported, use other sdk please", fieldDataList[0].GetType(), fieldDataList[0].GetFieldName()) } } else if ids != nil { - switch ids.IdField.(type) { + switch ids.GetIdField().(type) { case *schemapb.IDs_IntId: int64Pks := ids.GetIntId().GetData() rowsNum = int64(len(int64Pks)) @@ -1386,91 +1386,91 @@ func buildQueryResp(rowsNum int64, needFields []string, fieldDataList []*schemap row := map[string]interface{}{} if columnNum > 0 { for j := 0; j < columnNum; j++ { - switch fieldDataList[j].Type { + switch fieldDataList[j].GetType() { case schemapb.DataType_Bool: - if len(fieldDataList[j].ValidData) != 0 && !fieldDataList[j].ValidData[i] { - row[fieldDataList[j].FieldName] = nil + if len(fieldDataList[j].GetValidData()) != 0 && !fieldDataList[j].GetValidData()[i] { + row[fieldDataList[j].GetFieldName()] = nil continue } - row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetBoolData().Data[i] + row[fieldDataList[j].GetFieldName()] = fieldDataList[j].GetScalars().GetBoolData().GetData()[i] case schemapb.DataType_Int8: - if len(fieldDataList[j].ValidData) != 0 && !fieldDataList[j].ValidData[i] { - row[fieldDataList[j].FieldName] = nil + if len(fieldDataList[j].GetValidData()) != 0 && !fieldDataList[j].GetValidData()[i] { + row[fieldDataList[j].GetFieldName()] = nil continue } - row[fieldDataList[j].FieldName] = int8(fieldDataList[j].GetScalars().GetIntData().Data[i]) + row[fieldDataList[j].GetFieldName()] = int8(fieldDataList[j].GetScalars().GetIntData().GetData()[i]) case schemapb.DataType_Int16: - if len(fieldDataList[j].ValidData) != 0 && !fieldDataList[j].ValidData[i] { - row[fieldDataList[j].FieldName] = nil + if len(fieldDataList[j].GetValidData()) != 0 && !fieldDataList[j].GetValidData()[i] { + row[fieldDataList[j].GetFieldName()] = nil continue } - row[fieldDataList[j].FieldName] = int16(fieldDataList[j].GetScalars().GetIntData().Data[i]) + row[fieldDataList[j].GetFieldName()] = int16(fieldDataList[j].GetScalars().GetIntData().GetData()[i]) case schemapb.DataType_Int32: - if len(fieldDataList[j].ValidData) != 0 && !fieldDataList[j].ValidData[i] { - row[fieldDataList[j].FieldName] = nil + if len(fieldDataList[j].GetValidData()) != 0 && !fieldDataList[j].GetValidData()[i] { + row[fieldDataList[j].GetFieldName()] = nil continue } - row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetIntData().Data[i] + row[fieldDataList[j].GetFieldName()] = fieldDataList[j].GetScalars().GetIntData().GetData()[i] case schemapb.DataType_Int64: - if len(fieldDataList[j].ValidData) != 0 && !fieldDataList[j].ValidData[i] { - row[fieldDataList[j].FieldName] = nil + if len(fieldDataList[j].GetValidData()) != 0 && !fieldDataList[j].GetValidData()[i] { + row[fieldDataList[j].GetFieldName()] = nil continue } if enableInt64 { - row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetLongData().Data[i] + row[fieldDataList[j].GetFieldName()] = fieldDataList[j].GetScalars().GetLongData().GetData()[i] } else { - row[fieldDataList[j].FieldName] = strconv.FormatInt(fieldDataList[j].GetScalars().GetLongData().Data[i], 10) + row[fieldDataList[j].GetFieldName()] = strconv.FormatInt(fieldDataList[j].GetScalars().GetLongData().GetData()[i], 10) } case schemapb.DataType_Float: - if len(fieldDataList[j].ValidData) != 0 && !fieldDataList[j].ValidData[i] { - row[fieldDataList[j].FieldName] = nil + if len(fieldDataList[j].GetValidData()) != 0 && !fieldDataList[j].GetValidData()[i] { + row[fieldDataList[j].GetFieldName()] = nil continue } - row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetFloatData().Data[i] + row[fieldDataList[j].GetFieldName()] = fieldDataList[j].GetScalars().GetFloatData().GetData()[i] case schemapb.DataType_Double: - if len(fieldDataList[j].ValidData) != 0 && !fieldDataList[j].ValidData[i] { - row[fieldDataList[j].FieldName] = nil + if len(fieldDataList[j].GetValidData()) != 0 && !fieldDataList[j].GetValidData()[i] { + row[fieldDataList[j].GetFieldName()] = nil continue } - row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetDoubleData().Data[i] + row[fieldDataList[j].GetFieldName()] = fieldDataList[j].GetScalars().GetDoubleData().GetData()[i] case schemapb.DataType_String: - if len(fieldDataList[j].ValidData) != 0 && !fieldDataList[j].ValidData[i] { - row[fieldDataList[j].FieldName] = nil + if len(fieldDataList[j].GetValidData()) != 0 && !fieldDataList[j].GetValidData()[i] { + row[fieldDataList[j].GetFieldName()] = nil continue } - row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetStringData().Data[i] + row[fieldDataList[j].GetFieldName()] = fieldDataList[j].GetScalars().GetStringData().GetData()[i] case schemapb.DataType_VarChar: - if len(fieldDataList[j].ValidData) != 0 && !fieldDataList[j].ValidData[i] { - row[fieldDataList[j].FieldName] = nil + if len(fieldDataList[j].GetValidData()) != 0 && !fieldDataList[j].GetValidData()[i] { + row[fieldDataList[j].GetFieldName()] = nil continue } - row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetStringData().Data[i] + row[fieldDataList[j].GetFieldName()] = fieldDataList[j].GetScalars().GetStringData().GetData()[i] case schemapb.DataType_BinaryVector: - row[fieldDataList[j].FieldName] = fieldDataList[j].GetVectors().GetBinaryVector()[i*(fieldDataList[j].GetVectors().GetDim()/8) : (i+1)*(fieldDataList[j].GetVectors().GetDim()/8)] + row[fieldDataList[j].GetFieldName()] = fieldDataList[j].GetVectors().GetBinaryVector()[i*(fieldDataList[j].GetVectors().GetDim()/8) : (i+1)*(fieldDataList[j].GetVectors().GetDim()/8)] case schemapb.DataType_FloatVector: - row[fieldDataList[j].FieldName] = fieldDataList[j].GetVectors().GetFloatVector().Data[i*fieldDataList[j].GetVectors().GetDim() : (i+1)*fieldDataList[j].GetVectors().GetDim()] + row[fieldDataList[j].GetFieldName()] = fieldDataList[j].GetVectors().GetFloatVector().GetData()[i*fieldDataList[j].GetVectors().GetDim() : (i+1)*fieldDataList[j].GetVectors().GetDim()] case schemapb.DataType_Float16Vector: - row[fieldDataList[j].FieldName] = fieldDataList[j].GetVectors().GetFloat16Vector()[i*(fieldDataList[j].GetVectors().GetDim()*2) : (i+1)*(fieldDataList[j].GetVectors().GetDim()*2)] + row[fieldDataList[j].GetFieldName()] = fieldDataList[j].GetVectors().GetFloat16Vector()[i*(fieldDataList[j].GetVectors().GetDim()*2) : (i+1)*(fieldDataList[j].GetVectors().GetDim()*2)] case schemapb.DataType_BFloat16Vector: - row[fieldDataList[j].FieldName] = fieldDataList[j].GetVectors().GetBfloat16Vector()[i*(fieldDataList[j].GetVectors().GetDim()*2) : (i+1)*(fieldDataList[j].GetVectors().GetDim()*2)] + row[fieldDataList[j].GetFieldName()] = fieldDataList[j].GetVectors().GetBfloat16Vector()[i*(fieldDataList[j].GetVectors().GetDim()*2) : (i+1)*(fieldDataList[j].GetVectors().GetDim()*2)] case schemapb.DataType_SparseFloatVector: - row[fieldDataList[j].FieldName] = typeutil.SparseFloatBytesToMap(fieldDataList[j].GetVectors().GetSparseFloatVector().Contents[i]) + row[fieldDataList[j].GetFieldName()] = typeutil.SparseFloatBytesToMap(fieldDataList[j].GetVectors().GetSparseFloatVector().Contents[i]) case schemapb.DataType_Int8Vector: - row[fieldDataList[j].FieldName] = fieldDataList[j].GetVectors().GetInt8Vector()[i*fieldDataList[j].GetVectors().GetDim() : (i+1)*fieldDataList[j].GetVectors().GetDim()] + row[fieldDataList[j].GetFieldName()] = fieldDataList[j].GetVectors().GetInt8Vector()[i*fieldDataList[j].GetVectors().GetDim() : (i+1)*fieldDataList[j].GetVectors().GetDim()] case schemapb.DataType_Array: - if len(fieldDataList[j].ValidData) != 0 && !fieldDataList[j].ValidData[i] { - row[fieldDataList[j].FieldName] = nil + if len(fieldDataList[j].GetValidData()) != 0 && !fieldDataList[j].GetValidData()[i] { + row[fieldDataList[j].GetFieldName()] = nil continue } - row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetArrayData().Data[i] + row[fieldDataList[j].GetFieldName()] = fieldDataList[j].GetScalars().GetArrayData().GetData()[i] case schemapb.DataType_JSON: - if len(fieldDataList[j].ValidData) != 0 && !fieldDataList[j].ValidData[i] { - row[fieldDataList[j].FieldName] = nil + if len(fieldDataList[j].GetValidData()) != 0 && !fieldDataList[j].GetValidData()[i] { + row[fieldDataList[j].GetFieldName()] = nil continue } - data, ok := fieldDataList[j].GetScalars().Data.(*schemapb.ScalarField_JsonData) - if ok && !fieldDataList[j].IsDynamic { - row[fieldDataList[j].FieldName] = string(data.JsonData.Data[i]) + data, ok := fieldDataList[j].GetScalars().GetData().(*schemapb.ScalarField_JsonData) + if ok && !fieldDataList[j].GetIsDynamic() { + row[fieldDataList[j].GetFieldName()] = string(data.JsonData.GetData()[i]) } else { var dataMap map[string]interface{} @@ -1480,7 +1480,7 @@ func buildQueryResp(rowsNum int64, needFields []string, fieldDataList []*schemap return nil, err } - if containsString(dynamicOutputFields, fieldDataList[j].FieldName) { + if containsString(dynamicOutputFields, fieldDataList[j].GetFieldName()) { for key, value := range dataMap { row[key] = value } @@ -1493,12 +1493,12 @@ func buildQueryResp(rowsNum int64, needFields []string, fieldDataList []*schemap } } default: - row[fieldDataList[j].FieldName] = "" + row[fieldDataList[j].GetFieldName()] = "" } } } if ids != nil { - switch ids.IdField.(type) { + switch ids.GetIdField().(type) { case *schemapb.IDs_IntId: int64Pks := ids.GetIntId().GetData() if enableInt64 { diff --git a/internal/proxy/search_pipeline.go b/internal/proxy/search_pipeline.go index d4d88f6ff1..610c7f9fc2 100644 --- a/internal/proxy/search_pipeline.go +++ b/internal/proxy/search_pipeline.go @@ -561,12 +561,16 @@ func (op *filterFieldOperator) run(ctx context.Context, span trace.Span, inputs func mergeIDsFunc(ctx context.Context, span trace.Span, inputs ...any) ([]any, error) { multipleMilvusResults := inputs[0].([]*milvuspb.SearchResults) + idInt64Type := false idsList := lo.FilterMap(multipleMilvusResults, func(m *milvuspb.SearchResults, _ int) (*schemapb.IDs, bool) { + if m.GetResults().GetIds().GetIntId() != nil { + idInt64Type = true + } return m.Results.Ids, true }) + uniqueIDs := &schemapb.IDs{} - switch idsList[0].GetIdField().(type) { - case *schemapb.IDs_IntId: + if idInt64Type { idsSet := typeutil.NewSet[int64]() for _, ids := range idsList { if data := ids.GetIntId().GetData(); data != nil { @@ -578,7 +582,7 @@ func mergeIDsFunc(ctx context.Context, span trace.Span, inputs ...any) ([]any, e Data: idsSet.Collect(), }, } - case *schemapb.IDs_StrId: + } else { idsSet := typeutil.NewSet[string]() for _, ids := range idsList { if data := ids.GetStrId().GetData(); data != nil { diff --git a/internal/querynodev2/segments/result.go b/internal/querynodev2/segments/result.go index d1a0d83702..57b2a448ae 100644 --- a/internal/querynodev2/segments/result.go +++ b/internal/querynodev2/segments/result.go @@ -95,7 +95,9 @@ func ReduceSearchResults(ctx context.Context, results []*internalpb.SearchResult log.Debug("reduceSearchResultData", zap.Int("result No.", i), zap.Int64("nq", sData.NumQueries), - zap.Int64("topk", sData.TopK)) + zap.Int64("topk", sData.TopK), + zap.Int("ids.len", typeutil.GetSizeOfIDs(sData.Ids)), + zap.Int("fieldsData.len", len(sData.FieldsData))) } searchReduce := InitSearchReducer(info) @@ -235,7 +237,9 @@ func DecodeSearchResults(ctx context.Context, searchResults []*internalpb.Search return results, nil } -func EncodeSearchResultData(ctx context.Context, searchResultData *schemapb.SearchResultData, nq int64, topk int64, metricType string) (searchResults *internalpb.SearchResults, err error) { +func EncodeSearchResultData(ctx context.Context, searchResultData *schemapb.SearchResultData, + nq int64, topk int64, metricType string, +) (searchResults *internalpb.SearchResults, err error) { _, sp := otel.Tracer(typeutil.QueryNodeRole).Start(ctx, "EncodeSearchResultData") defer sp.End()