admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-07 17:48:29 +08:00
Code Issues Packages Projects Releases Wiki Activity

add a custom http header: Accept-Type-Allow-Int64 (#27901)

Browse Source 
Signed-off-by: PowderLi <min.li@zilliz.com>
This commit is contained in:
PowderLi 2023-11-01 11:42:16 +08:00 committed by GitHub
parent 0c69f48ba4
commit 0c0f012e03
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 763 additions and 409 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
internal/core/src/storage/MinioChunkManager.cpp
View File

@ -347,7 +347,7 @@ MinioChunkManager::Remove(const std::string& filepath) {
std::vector<std::string> std::vector<std::string>
MinioChunkManager::ListWithPrefix(const std::string& filepath) { MinioChunkManager::ListWithPrefix(const std::string& filepath) {
return ListObjects(default_bucket_name_.c_str(), filepath.c_str()); return ListObjects(default_bucket_name_, filepath);
} }
uint64_t uint64_t
@ -393,7 +393,7 @@ MinioChunkManager::ListBuckets() {
ThrowS3Error("ListBuckets", err, "params"); ThrowS3Error("ListBuckets", err, "params");
} }
for (auto&& b : outcome.GetResult().GetBuckets()) { for (auto&& b : outcome.GetResult().GetBuckets()) {
buckets.emplace_back(b.GetName().c_str()); buckets.emplace_back(b.GetName());
} }
return buckets; return buckets;
} }
@ -623,7 +623,7 @@ MinioChunkManager::ListObjects(const std::string& bucket_name,
} }
auto objects = outcome.GetResult().GetContents(); auto objects = outcome.GetResult().GetContents();
for (auto& obj : objects) { for (auto& obj : objects) {
objects_vec.emplace_back(obj.GetKey().c_str()); objects_vec.emplace_back(obj.GetKey());
} }
return objects_vec; return objects_vec;
} }

2
internal/distributed/proxy/httpserver/constant.go
View File

@ -24,7 +24,7 @@ const (
DefaultDbName = "default" DefaultDbName = "default"
DefaultIndexName = "vector_idx" DefaultIndexName = "vector_idx"
DefaultOutputFields = "*" DefaultOutputFields = "*"
HTTPHeaderAllowInt64 = "Accept-Type-Allow-Int64"
HTTPReturnCode = "code" HTTPReturnCode = "code"
HTTPReturnMessage = "message" HTTPReturnMessage = "message"
HTTPReturnData = "data" HTTPReturnData = "data"

183
internal/distributed/proxy/httpserver/handler_v1.go
View File

@ -54,7 +54,10 @@ func (h *Handlers) checkDatabase(ctx context.Context, c *gin.Context, dbName str
return true return true
} }
} }
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrDatabaseNotFound), HTTPReturnMessage: merr.ErrDatabaseNotFound.Error()}) c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrDatabaseNotFound),
HTTPReturnMessage: merr.ErrDatabaseNotFound.Error() + ", database: " + dbName,
})
return false return false
} }
@ -152,12 +155,18 @@ func (h *Handlers) createCollection(c *gin.Context) {
} }
if err := c.ShouldBindBodyWith(&httpReq, binding.JSON); err != nil { if err := c.ShouldBindBodyWith(&httpReq, binding.JSON); err != nil {
log.Warn("high level restful api, the parameter of create collection is incorrect", zap.Any("request", httpReq), zap.Error(err)) log.Warn("high level restful api, the parameter of create collection is incorrect", zap.Any("request", httpReq), zap.Error(err))
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrIncorrectParameterFormat), HTTPReturnMessage: merr.ErrIncorrectParameterFormat.Error()}) c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrIncorrectParameterFormat),
HTTPReturnMessage: merr.ErrIncorrectParameterFormat.Error() + ", error: " + err.Error(),
})
return return
} }
if httpReq.CollectionName == "" || httpReq.Dimension == 0 { if httpReq.CollectionName == "" || httpReq.Dimension == 0 {
log.Warn("high level restful api, create collection require parameters: [collectionName, dimension], but miss", zap.Any("request", httpReq)) log.Warn("high level restful api, create collection require parameters: [collectionName, dimension], but miss", zap.Any("request", httpReq))
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrMissingRequiredParameters), HTTPReturnMessage: merr.ErrMissingRequiredParameters.Error()}) c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrMissingRequiredParameters),
HTTPReturnMessage: merr.ErrMissingRequiredParameters.Error() + ", required parameters: [collectionName, dimension]",
})
return return
} }
schema, err := proto.Marshal(&schemapb.CollectionSchema{ schema, err := proto.Marshal(&schemapb.CollectionSchema{
@ -189,7 +198,10 @@ func (h *Handlers) createCollection(c *gin.Context) {
}) })
if err != nil { if err != nil {
log.Warn("high level restful api, marshal collection schema fail", zap.Any("request", httpReq), zap.Error(err)) log.Warn("high level restful api, marshal collection schema fail", zap.Any("request", httpReq), zap.Error(err))
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrMarshalCollectionSchema), HTTPReturnMessage: merr.ErrMarshalCollectionSchema.Error()}) c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrMarshalCollectionSchema),
HTTPReturnMessage: merr.ErrMarshalCollectionSchema.Error() + ", error: " + err.Error(),
})
return return
} }
req := milvuspb.CreateCollectionRequest{ req := milvuspb.CreateCollectionRequest{
@ -248,7 +260,10 @@ func (h *Handlers) getCollectionDetails(c *gin.Context) {
collectionName := c.Query(HTTPCollectionName) collectionName := c.Query(HTTPCollectionName)
if collectionName == "" { if collectionName == "" {
log.Warn("high level restful api, desc collection require parameter: [collectionName], but miss") log.Warn("high level restful api, desc collection require parameter: [collectionName], but miss")
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrMissingRequiredParameters), HTTPReturnMessage: merr.ErrMissingRequiredParameters.Error()}) c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrMissingRequiredParameters),
HTTPReturnMessage: merr.ErrMissingRequiredParameters.Error() + ", required parameters: [collectionName]",
})
return return
} }
dbName := c.DefaultQuery(HTTPDbName, DefaultDbName) dbName := c.DefaultQuery(HTTPDbName, DefaultDbName)
@ -320,12 +335,18 @@ func (h *Handlers) dropCollection(c *gin.Context) {
} }
if err := c.ShouldBindBodyWith(&httpReq, binding.JSON); err != nil { if err := c.ShouldBindBodyWith(&httpReq, binding.JSON); err != nil {
log.Warn("high level restful api, the parameter of drop collection is incorrect", zap.Any("request", httpReq), zap.Error(err)) log.Warn("high level restful api, the parameter of drop collection is incorrect", zap.Any("request", httpReq), zap.Error(err))
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrIncorrectParameterFormat), HTTPReturnMessage: merr.ErrIncorrectParameterFormat.Error()}) c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrIncorrectParameterFormat),
HTTPReturnMessage: merr.ErrIncorrectParameterFormat.Error() + ", error: " + err.Error(),
})
return return
} }
if httpReq.CollectionName == "" { if httpReq.CollectionName == "" {
log.Warn("high level restful api, drop collection require parameter: [collectionName], but miss") log.Warn("high level restful api, drop collection require parameter: [collectionName], but miss")
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrMissingRequiredParameters), HTTPReturnMessage: merr.ErrMissingRequiredParameters.Error()}) c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrMissingRequiredParameters),
HTTPReturnMessage: merr.ErrMissingRequiredParameters.Error() + ", required parameters: [collectionName]",
})
return return
} }
req := milvuspb.DropCollectionRequest{ req := milvuspb.DropCollectionRequest{
@ -345,7 +366,10 @@ func (h *Handlers) dropCollection(c *gin.Context) {
return return
} }
if !has { if !has {
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrCollectionNotFound), HTTPReturnMessage: merr.ErrCollectionNotFound.Error()}) c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrCollectionNotFound),
HTTPReturnMessage: merr.ErrCollectionNotFound.Error() + ", database: " + httpReq.DbName + ", collection: " + httpReq.CollectionName,
})
return return
} }
response, err := h.proxy.DropCollection(ctx, &req) response, err := h.proxy.DropCollection(ctx, &req)
@ -367,12 +391,18 @@ func (h *Handlers) query(c *gin.Context) {
} }
if err := c.ShouldBindBodyWith(&httpReq, binding.JSON); err != nil { if err := c.ShouldBindBodyWith(&httpReq, binding.JSON); err != nil {
log.Warn("high level restful api, the parameter of query is incorrect", zap.Any("request", httpReq), zap.Error(err)) log.Warn("high level restful api, the parameter of query is incorrect", zap.Any("request", httpReq), zap.Error(err))
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrIncorrectParameterFormat), HTTPReturnMessage: merr.ErrIncorrectParameterFormat.Error()}) c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrIncorrectParameterFormat),
HTTPReturnMessage: merr.ErrIncorrectParameterFormat.Error() + ", error: " + err.Error(),
})
return return
} }
if httpReq.CollectionName == "" || httpReq.Filter == "" { if httpReq.CollectionName == "" || httpReq.Filter == "" {
log.Warn("high level restful api, query require parameter: [collectionName, filter], but miss") log.Warn("high level restful api, query require parameter: [collectionName, filter], but miss")
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrMissingRequiredParameters), HTTPReturnMessage: merr.ErrMissingRequiredParameters.Error()}) c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrMissingRequiredParameters),
HTTPReturnMessage: merr.ErrMissingRequiredParameters.Error() + ", required parameters: [collectionName, filter]",
})
return return
} }
req := milvuspb.QueryRequest{ req := milvuspb.QueryRequest{
@ -404,10 +434,14 @@ func (h *Handlers) query(c *gin.Context) {
if err != nil { if err != nil {
c.JSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(err), HTTPReturnMessage: err.Error()}) c.JSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(err), HTTPReturnMessage: err.Error()})
} else { } else {
outputData, err := buildQueryResp(int64(0), response.OutputFields, response.FieldsData, nil, nil) allowJS, _ := strconv.ParseBool(c.Request.Header.Get(HTTPHeaderAllowInt64))
outputData, err := buildQueryResp(int64(0), response.OutputFields, response.FieldsData, nil, nil, allowJS)
if err != nil { if err != nil {
log.Warn("high level restful api, fail to deal with query result", zap.Any("response", response), zap.Error(err)) log.Warn("high level restful api, fail to deal with query result", zap.Any("response", response), zap.Error(err))
c.JSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrInvalidSearchResult), HTTPReturnMessage: merr.ErrInvalidSearchResult.Error()}) c.JSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrInvalidSearchResult),
HTTPReturnMessage: merr.ErrInvalidSearchResult.Error() + ", error: " + err.Error(),
})
} else { } else {
c.JSON(http.StatusOK, gin.H{HTTPReturnCode: http.StatusOK, HTTPReturnData: outputData}) c.JSON(http.StatusOK, gin.H{HTTPReturnCode: http.StatusOK, HTTPReturnData: outputData})
} }
@ -421,12 +455,18 @@ func (h *Handlers) get(c *gin.Context) {
} }
if err := c.ShouldBindBodyWith(&httpReq, binding.JSON); err != nil { if err := c.ShouldBindBodyWith(&httpReq, binding.JSON); err != nil {
log.Warn("high level restful api, the parameter of get is incorrect", zap.Any("request", httpReq), zap.Error(err)) log.Warn("high level restful api, the parameter of get is incorrect", zap.Any("request", httpReq), zap.Error(err))
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrIncorrectParameterFormat), HTTPReturnMessage: merr.ErrIncorrectParameterFormat.Error()}) c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrIncorrectParameterFormat),
HTTPReturnMessage: merr.ErrIncorrectParameterFormat.Error() + ", error: " + err.Error(),
})
return return
} }
if httpReq.CollectionName == "" || httpReq.ID == nil { if httpReq.CollectionName == "" || httpReq.ID == nil {
log.Warn("high level restful api, get require parameter: [collectionName, id], but miss") log.Warn("high level restful api, get require parameter: [collectionName, id], but miss")
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrMissingRequiredParameters), HTTPReturnMessage: merr.ErrMissingRequiredParameters.Error()}) c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrMissingRequiredParameters),
HTTPReturnMessage: merr.ErrMissingRequiredParameters.Error() + ", required parameters: [collectionName, id]",
})
return return
} }
req := milvuspb.QueryRequest{ req := milvuspb.QueryRequest{
@ -450,7 +490,10 @@ func (h *Handlers) get(c *gin.Context) {
body, _ := c.Get(gin.BodyBytesKey) body, _ := c.Get(gin.BodyBytesKey)
filter, err := checkGetPrimaryKey(coll.Schema, gjson.Get(string(body.([]byte)), DefaultPrimaryFieldName)) filter, err := checkGetPrimaryKey(coll.Schema, gjson.Get(string(body.([]byte)), DefaultPrimaryFieldName))
if err != nil { if err != nil {
c.JSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrCheckPrimaryKey), HTTPReturnMessage: merr.ErrCheckPrimaryKey.Error()}) c.JSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrCheckPrimaryKey),
HTTPReturnMessage: merr.ErrCheckPrimaryKey.Error() + ", error: " + err.Error(),
})
return return
} }
req.Expr = filter req.Expr = filter
@ -461,13 +504,16 @@ func (h *Handlers) get(c *gin.Context) {
if err != nil { if err != nil {
c.JSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(err), HTTPReturnMessage: err.Error()}) c.JSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(err), HTTPReturnMessage: err.Error()})
} else { } else {
outputData, err := buildQueryResp(int64(0), response.OutputFields, response.FieldsData, nil, nil) allowJS, _ := strconv.ParseBool(c.Request.Header.Get(HTTPHeaderAllowInt64))
outputData, err := buildQueryResp(int64(0), response.OutputFields, response.FieldsData, nil, nil, allowJS)
if err != nil { if err != nil {
log.Warn("high level restful api, fail to deal with get result", zap.Any("response", response), zap.Error(err)) log.Warn("high level restful api, fail to deal with get result", zap.Any("response", response), zap.Error(err))
c.JSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrInvalidSearchResult), HTTPReturnMessage: merr.ErrInvalidSearchResult.Error()}) c.JSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrInvalidSearchResult),
HTTPReturnMessage: merr.ErrInvalidSearchResult.Error() + ", error: " + err.Error(),
})
} else { } else {
c.JSON(http.StatusOK, gin.H{HTTPReturnCode: http.StatusOK, HTTPReturnData: outputData}) c.JSON(http.StatusOK, gin.H{HTTPReturnCode: http.StatusOK, HTTPReturnData: outputData})
log.Error("get resultIS: ", zap.Any("res", outputData))
} }
} }
} }
@ -478,12 +524,18 @@ func (h *Handlers) delete(c *gin.Context) {
} }
if err := c.ShouldBindBodyWith(&httpReq, binding.JSON); err != nil { if err := c.ShouldBindBodyWith(&httpReq, binding.JSON); err != nil {
log.Warn("high level restful api, the parameter of delete is incorrect", zap.Any("request", httpReq), zap.Error(err)) log.Warn("high level restful api, the parameter of delete is incorrect", zap.Any("request", httpReq), zap.Error(err))
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrIncorrectParameterFormat), HTTPReturnMessage: merr.ErrIncorrectParameterFormat.Error()}) c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrIncorrectParameterFormat),
HTTPReturnMessage: merr.ErrIncorrectParameterFormat.Error() + ", error: " + err.Error(),
})
return return
} }
if httpReq.CollectionName == "" || (httpReq.ID == nil && httpReq.Filter == "") { if httpReq.CollectionName == "" || (httpReq.ID == nil && httpReq.Filter == "") {
log.Warn("high level restful api, delete require parameter: [collectionName, id/filter], but miss") log.Warn("high level restful api, delete require parameter: [collectionName, id/filter], but miss")
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrMissingRequiredParameters), HTTPReturnMessage: merr.ErrMissingRequiredParameters.Error()}) c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrMissingRequiredParameters),
HTTPReturnMessage: merr.ErrMissingRequiredParameters.Error() + ", required parameters: [collectionName, id/filter]",
})
return return
} }
req := milvuspb.DeleteRequest{ req := milvuspb.DeleteRequest{
@ -507,7 +559,10 @@ func (h *Handlers) delete(c *gin.Context) {
body, _ := c.Get(gin.BodyBytesKey) body, _ := c.Get(gin.BodyBytesKey)
filter, err := checkGetPrimaryKey(coll.Schema, gjson.Get(string(body.([]byte)), DefaultPrimaryFieldName)) filter, err := checkGetPrimaryKey(coll.Schema, gjson.Get(string(body.([]byte)), DefaultPrimaryFieldName))
if err != nil { if err != nil {
c.JSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrCheckPrimaryKey), HTTPReturnMessage: merr.ErrCheckPrimaryKey.Error()}) c.JSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrCheckPrimaryKey),
HTTPReturnMessage: merr.ErrCheckPrimaryKey.Error() + ", error: " + err.Error(),
})
return return
} }
req.Expr = filter req.Expr = filter
@ -533,7 +588,10 @@ func (h *Handlers) insert(c *gin.Context) {
} }
if err = c.ShouldBindBodyWith(&singleInsertReq, binding.JSON); err != nil { if err = c.ShouldBindBodyWith(&singleInsertReq, binding.JSON); err != nil {
log.Warn("high level restful api, the parameter of insert is incorrect", zap.Any("request", httpReq), zap.Error(err)) log.Warn("high level restful api, the parameter of insert is incorrect", zap.Any("request", httpReq), zap.Error(err))
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrIncorrectParameterFormat), HTTPReturnMessage: merr.ErrIncorrectParameterFormat.Error()}) c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrIncorrectParameterFormat),
HTTPReturnMessage: merr.ErrIncorrectParameterFormat.Error() + ", error: " + err.Error(),
})
return return
} }
httpReq.DbName = singleInsertReq.DbName httpReq.DbName = singleInsertReq.DbName
@ -542,7 +600,10 @@ func (h *Handlers) insert(c *gin.Context) {
} }
if httpReq.CollectionName == "" || httpReq.Data == nil { if httpReq.CollectionName == "" || httpReq.Data == nil {
log.Warn("high level restful api, insert require parameter: [collectionName, data], but miss") log.Warn("high level restful api, insert require parameter: [collectionName, data], but miss")
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrMissingRequiredParameters), HTTPReturnMessage: merr.ErrMissingRequiredParameters.Error()}) c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrMissingRequiredParameters),
HTTPReturnMessage: merr.ErrMissingRequiredParameters.Error() + ", required parameters: [collectionName, data]",
})
return return
} }
req := milvuspb.InsertRequest{ req := milvuspb.InsertRequest{
@ -567,13 +628,19 @@ func (h *Handlers) insert(c *gin.Context) {
err, httpReq.Data = checkAndSetData(string(body.([]byte)), coll) err, httpReq.Data = checkAndSetData(string(body.([]byte)), coll)
if err != nil { if err != nil {
log.Warn("high level restful api, fail to deal with insert data", zap.Any("body", body), zap.Error(err)) log.Warn("high level restful api, fail to deal with insert data", zap.Any("body", body), zap.Error(err))
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrInvalidInsertData), HTTPReturnMessage: merr.ErrInvalidInsertData.Error()}) c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrInvalidInsertData),
HTTPReturnMessage: merr.ErrInvalidInsertData.Error() + ", error: " + err.Error(),
})
return return
} }
req.FieldsData, err = anyToColumns(httpReq.Data, coll.Schema) req.FieldsData, err = anyToColumns(httpReq.Data, coll.Schema)
if err != nil { if err != nil {
log.Warn("high level restful api, fail to deal with insert data", zap.Any("data", httpReq.Data), zap.Error(err)) log.Warn("high level restful api, fail to deal with insert data", zap.Any("data", httpReq.Data), zap.Error(err))
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrInvalidInsertData), HTTPReturnMessage: merr.ErrInvalidInsertData.Error()}) c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrInvalidInsertData),
HTTPReturnMessage: merr.ErrInvalidInsertData.Error() + ", error: " + err.Error(),
})
return return
} }
response, err := h.proxy.Insert(ctx, &req) response, err := h.proxy.Insert(ctx, &req)
@ -585,11 +652,19 @@ func (h *Handlers) insert(c *gin.Context) {
} else { } else {
switch response.IDs.GetIdField().(type) { switch response.IDs.GetIdField().(type) {
case *schemapb.IDs_IntId: case *schemapb.IDs_IntId:
allowJS, _ := strconv.ParseBool(c.Request.Header.Get(HTTPHeaderAllowInt64))
if allowJS {
c.JSON(http.StatusOK, gin.H{HTTPReturnCode: http.StatusOK, HTTPReturnData: gin.H{"insertCount": response.InsertCnt, "insertIds": response.IDs.IdField.(*schemapb.IDs_IntId).IntId.Data}}) c.JSON(http.StatusOK, gin.H{HTTPReturnCode: http.StatusOK, HTTPReturnData: gin.H{"insertCount": response.InsertCnt, "insertIds": response.IDs.IdField.(*schemapb.IDs_IntId).IntId.Data}})
} else {
c.JSON(http.StatusOK, gin.H{HTTPReturnCode: http.StatusOK, HTTPReturnData: gin.H{"insertCount": response.InsertCnt, "insertIds": formatInt64(response.IDs.IdField.(*schemapb.IDs_IntId).IntId.Data)}})
}
case *schemapb.IDs_StrId: case *schemapb.IDs_StrId:
c.JSON(http.StatusOK, gin.H{HTTPReturnCode: http.StatusOK, HTTPReturnData: gin.H{"insertCount": response.InsertCnt, "insertIds": response.IDs.IdField.(*schemapb.IDs_StrId).StrId.Data}}) c.JSON(http.StatusOK, gin.H{HTTPReturnCode: http.StatusOK, HTTPReturnData: gin.H{"insertCount": response.InsertCnt, "insertIds": response.IDs.IdField.(*schemapb.IDs_StrId).StrId.Data}})
default: default:
c.JSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrCheckPrimaryKey), HTTPReturnMessage: merr.ErrCheckPrimaryKey.Error()}) c.JSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrCheckPrimaryKey),
HTTPReturnMessage: merr.ErrCheckPrimaryKey.Error() + ", error: unsupported primary key data type",
})
} }
} }
} }
@ -603,8 +678,11 @@ func (h *Handlers) upsert(c *gin.Context) {
DbName: DefaultDbName, DbName: DefaultDbName,
} }
if err = c.ShouldBindBodyWith(&singleUpsertReq, binding.JSON); err != nil { if err = c.ShouldBindBodyWith(&singleUpsertReq, binding.JSON); err != nil {
log.Warn("high level restful api, the parameter of insert is incorrect", zap.Any("request", httpReq), zap.Error(err)) log.Warn("high level restful api, the parameter of upsert is incorrect", zap.Any("request", httpReq), zap.Error(err))
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrIncorrectParameterFormat), HTTPReturnMessage: merr.ErrIncorrectParameterFormat.Error()}) c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrIncorrectParameterFormat),
HTTPReturnMessage: merr.ErrIncorrectParameterFormat.Error() + ", error: " + err.Error(),
})
return return
} }
httpReq.DbName = singleUpsertReq.DbName httpReq.DbName = singleUpsertReq.DbName
@ -612,8 +690,11 @@ func (h *Handlers) upsert(c *gin.Context) {
httpReq.Data = []map[string]interface{}{singleUpsertReq.Data} httpReq.Data = []map[string]interface{}{singleUpsertReq.Data}
} }
if httpReq.CollectionName == "" || httpReq.Data == nil { if httpReq.CollectionName == "" || httpReq.Data == nil {
log.Warn("high level restful api, insert require parameter: [collectionName, data], but miss") log.Warn("high level restful api, upsert require parameter: [collectionName, data], but miss")
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrMissingRequiredParameters), HTTPReturnMessage: merr.ErrMissingRequiredParameters.Error()}) c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrMissingRequiredParameters),
HTTPReturnMessage: merr.ErrMissingRequiredParameters.Error() + ", required parameters: [collectionName, data]",
})
return return
} }
req := milvuspb.UpsertRequest{ req := milvuspb.UpsertRequest{
@ -642,14 +723,20 @@ func (h *Handlers) upsert(c *gin.Context) {
body, _ := c.Get(gin.BodyBytesKey) body, _ := c.Get(gin.BodyBytesKey)
err, httpReq.Data = checkAndSetData(string(body.([]byte)), coll) err, httpReq.Data = checkAndSetData(string(body.([]byte)), coll)
if err != nil { if err != nil {
log.Warn("high level restful api, fail to deal with insert data", zap.Any("body", body), zap.Error(err)) log.Warn("high level restful api, fail to deal with upsert data", zap.Any("body", body), zap.Error(err))
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrInvalidInsertData), HTTPReturnMessage: merr.ErrInvalidInsertData.Error()}) c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrInvalidInsertData),
HTTPReturnMessage: merr.ErrInvalidInsertData.Error() + ", error: " + err.Error(),
})
return return
} }
req.FieldsData, err = anyToColumns(httpReq.Data, coll.Schema) req.FieldsData, err = anyToColumns(httpReq.Data, coll.Schema)
if err != nil { if err != nil {
log.Warn("high level restful api, fail to deal with insert data", zap.Any("data", httpReq.Data), zap.Error(err)) log.Warn("high level restful api, fail to deal with upsert data", zap.Any("data", httpReq.Data), zap.Error(err))
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrInvalidInsertData), HTTPReturnMessage: merr.ErrInvalidInsertData.Error()}) c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrInvalidInsertData),
HTTPReturnMessage: merr.ErrInvalidInsertData.Error() + ", error: " + err.Error(),
})
return return
} }
response, err := h.proxy.Upsert(ctx, &req) response, err := h.proxy.Upsert(ctx, &req)
@ -661,11 +748,19 @@ func (h *Handlers) upsert(c *gin.Context) {
} else { } else {
switch response.IDs.GetIdField().(type) { switch response.IDs.GetIdField().(type) {
case *schemapb.IDs_IntId: case *schemapb.IDs_IntId:
allowJS, _ := strconv.ParseBool(c.Request.Header.Get(HTTPHeaderAllowInt64))
if allowJS {
c.JSON(http.StatusOK, gin.H{HTTPReturnCode: http.StatusOK, HTTPReturnData: gin.H{"upsertCount": response.UpsertCnt, "upsertIds": response.IDs.IdField.(*schemapb.IDs_IntId).IntId.Data}}) c.JSON(http.StatusOK, gin.H{HTTPReturnCode: http.StatusOK, HTTPReturnData: gin.H{"upsertCount": response.UpsertCnt, "upsertIds": response.IDs.IdField.(*schemapb.IDs_IntId).IntId.Data}})
} else {
c.JSON(http.StatusOK, gin.H{HTTPReturnCode: http.StatusOK, HTTPReturnData: gin.H{"upsertCount": response.UpsertCnt, "upsertIds": formatInt64(response.IDs.IdField.(*schemapb.IDs_IntId).IntId.Data)}})
}
case *schemapb.IDs_StrId: case *schemapb.IDs_StrId:
c.JSON(http.StatusOK, gin.H{HTTPReturnCode: http.StatusOK, HTTPReturnData: gin.H{"upsertCount": response.UpsertCnt, "upsertIds": response.IDs.IdField.(*schemapb.IDs_StrId).StrId.Data}}) c.JSON(http.StatusOK, gin.H{HTTPReturnCode: http.StatusOK, HTTPReturnData: gin.H{"upsertCount": response.UpsertCnt, "upsertIds": response.IDs.IdField.(*schemapb.IDs_StrId).StrId.Data}})
default: default:
c.JSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrCheckPrimaryKey), HTTPReturnMessage: merr.ErrCheckPrimaryKey.Error()}) c.JSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrCheckPrimaryKey),
HTTPReturnMessage: merr.ErrCheckPrimaryKey.Error() + ", error: unsupported primary key data type",
})
} }
} }
} }
@ -677,12 +772,18 @@ func (h *Handlers) search(c *gin.Context) {
} }
if err := c.ShouldBindBodyWith(&httpReq, binding.JSON); err != nil { if err := c.ShouldBindBodyWith(&httpReq, binding.JSON); err != nil {
log.Warn("high level restful api, the parameter of search is incorrect", zap.Any("request", httpReq), zap.Error(err)) log.Warn("high level restful api, the parameter of search is incorrect", zap.Any("request", httpReq), zap.Error(err))
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrIncorrectParameterFormat), HTTPReturnMessage: merr.ErrIncorrectParameterFormat.Error()}) c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrIncorrectParameterFormat),
HTTPReturnMessage: merr.ErrIncorrectParameterFormat.Error() + ", error: " + err.Error(),
})
return return
} }
if httpReq.CollectionName == "" || httpReq.Vector == nil { if httpReq.CollectionName == "" || httpReq.Vector == nil {
log.Warn("high level restful api, search require parameter: [collectionName, vector], but miss") log.Warn("high level restful api, search require parameter: [collectionName, vector], but miss")
c.AbortWithStatusJSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrMissingRequiredParameters), HTTPReturnMessage: merr.ErrMissingRequiredParameters.Error()}) c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrMissingRequiredParameters),
HTTPReturnMessage: merr.ErrMissingRequiredParameters.Error() + ", required parameters: [collectionName, vector]",
})
return return
} }
params := map[string]interface{}{ // auto generated mapping params := map[string]interface{}{ // auto generated mapping
@ -724,10 +825,14 @@ func (h *Handlers) search(c *gin.Context) {
if response.Results.TopK == int64(0) { if response.Results.TopK == int64(0) {
c.JSON(http.StatusOK, gin.H{HTTPReturnCode: http.StatusOK, HTTPReturnData: []interface{}{}}) c.JSON(http.StatusOK, gin.H{HTTPReturnCode: http.StatusOK, HTTPReturnData: []interface{}{}})
} else { } else {
outputData, err := buildQueryResp(response.Results.TopK, response.Results.OutputFields, response.Results.FieldsData, response.Results.Ids, response.Results.Scores) allowJS, _ := strconv.ParseBool(c.Request.Header.Get(HTTPHeaderAllowInt64))
outputData, err := buildQueryResp(response.Results.TopK, response.Results.OutputFields, response.Results.FieldsData, response.Results.Ids, response.Results.Scores, allowJS)
if err != nil { if err != nil {
log.Warn("high level restful api, fail to deal with search result", zap.Any("result", response.Results), zap.Error(err)) log.Warn("high level restful api, fail to deal with search result", zap.Any("result", response.Results), zap.Error(err))
c.JSON(http.StatusOK, gin.H{HTTPReturnCode: merr.Code(merr.ErrInvalidSearchResult), HTTPReturnMessage: merr.ErrInvalidSearchResult.Error()}) c.JSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrInvalidSearchResult),
HTTPReturnMessage: merr.ErrInvalidSearchResult.Error() + ", error: " + err.Error(),
})
} else { } else {
c.JSON(http.StatusOK, gin.H{HTTPReturnCode: http.StatusOK, HTTPReturnData: outputData}) c.JSON(http.StatusOK, gin.H{HTTPReturnCode: http.StatusOK, HTTPReturnData: outputData})
} }

737
internal/distributed/proxy/httpserver/handler_v1_test.go
View File

File diff suppressed because it is too large Load Diff

18
internal/distributed/proxy/httpserver/utils.go
View File

@ -724,7 +724,7 @@ func genDynamicFields(fields []string, list []*schemapb.FieldData) []string {
return dynamicFields return dynamicFields
} }
func buildQueryResp(rowsNum int64, needFields []string, fieldDataList []*schemapb.FieldData, ids *schemapb.IDs, scores []float32) ([]map[string]interface{}, error) { func buildQueryResp(rowsNum int64, needFields []string, fieldDataList []*schemapb.FieldData, ids *schemapb.IDs, scores []float32, enableInt64 bool) ([]map[string]interface{}, error) {
var queryResp []map[string]interface{} var queryResp []map[string]interface{}
columnNum := len(fieldDataList) columnNum := len(fieldDataList)
@ -791,7 +791,11 @@ func buildQueryResp(rowsNum int64, needFields []string, fieldDataList []*schemap
case schemapb.DataType_Int32: case schemapb.DataType_Int32:
row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetIntData().Data[i] row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetIntData().Data[i]
case schemapb.DataType_Int64: case schemapb.DataType_Int64:
if enableInt64 {
row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetLongData().Data[i] row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetLongData().Data[i]
} else {
row[fieldDataList[j].FieldName] = strconv.FormatInt(fieldDataList[j].GetScalars().GetLongData().Data[i], 10)
}
case schemapb.DataType_Float: case schemapb.DataType_Float:
row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetFloatData().Data[i] row[fieldDataList[j].FieldName] = fieldDataList[j].GetScalars().GetFloatData().Data[i]
case schemapb.DataType_Double: case schemapb.DataType_Double:
@ -840,7 +844,11 @@ func buildQueryResp(rowsNum int64, needFields []string, fieldDataList []*schemap
switch ids.IdField.(type) { switch ids.IdField.(type) {
case *schemapb.IDs_IntId: case *schemapb.IDs_IntId:
int64Pks := ids.GetIntId().GetData() int64Pks := ids.GetIntId().GetData()
if enableInt64 {
row[DefaultPrimaryFieldName] = int64Pks[i] row[DefaultPrimaryFieldName] = int64Pks[i]
} else {
row[DefaultPrimaryFieldName] = strconv.FormatInt(int64Pks[i], 10)
}
case *schemapb.IDs_StrId: case *schemapb.IDs_StrId:
stringPks := ids.GetStrId().GetData() stringPks := ids.GetStrId().GetData()
row[DefaultPrimaryFieldName] = stringPks[i] row[DefaultPrimaryFieldName] = stringPks[i]
@ -856,3 +864,11 @@ func buildQueryResp(rowsNum int64, needFields []string, fieldDataList []*schemap
return queryResp, nil return queryResp, nil
} }
func formatInt64(intArray []int64) []string {
stringArray := make([]string, 0)
for _, i := range intArray {
stringArray = append(stringArray, strconv.FormatInt(i, 10))
}
return stringArray
}

147
internal/distributed/proxy/httpserver/utils_test.go
View File

@ -31,7 +31,7 @@ func generatePrimaryField(datatype schemapb.DataType) schemapb.FieldSchema {
} }
} }
func generateIds(num int) *schemapb.IDs { func generateIds(dataType schemapb.DataType, num int) *schemapb.IDs {
var intArray []int64 var intArray []int64
if num == 0 { if num == 0 {
intArray = []int64{} intArray = []int64{}
@ -40,6 +40,8 @@ func generateIds(num int) *schemapb.IDs {
intArray = append(intArray, i) intArray = append(intArray, i)
} }
} }
switch dataType {
case schemapb.DataType_Int64:
return &schemapb.IDs{ return &schemapb.IDs{
IdField: &schemapb.IDs_IntId{ IdField: &schemapb.IDs_IntId{
IntId: &schemapb.LongArray{ IntId: &schemapb.LongArray{
@ -47,6 +49,17 @@ func generateIds(num int) *schemapb.IDs {
}, },
}, },
} }
case schemapb.DataType_VarChar:
stringArray := formatInt64(intArray)
return &schemapb.IDs{
IdField: &schemapb.IDs_StrId{
StrId: &schemapb.StringArray{
Data: stringArray,
},
},
}
}
return nil
} }
func generateVectorFieldSchema(useBinary bool) schemapb.FieldSchema { func generateVectorFieldSchema(useBinary bool) schemapb.FieldSchema {
@ -82,8 +95,8 @@ func generateVectorFieldSchema(useBinary bool) schemapb.FieldSchema {
} }
} }
func generateCollectionSchema(useBinary bool) *schemapb.CollectionSchema { func generateCollectionSchema(datatype schemapb.DataType, useBinary bool) *schemapb.CollectionSchema {
primaryField := generatePrimaryField(schemapb.DataType_Int64) primaryField := generatePrimaryField(datatype)
vectorField := generateVectorFieldSchema(useBinary) vectorField := generateVectorFieldSchema(useBinary)
return &schemapb.CollectionSchema{ return &schemapb.CollectionSchema{
Name: DefaultCollectionName, Name: DefaultCollectionName,
@ -196,7 +209,7 @@ func generateFieldData() []*schemapb.FieldData {
return []*schemapb.FieldData{&fieldData1, &fieldData2, &fieldData3} return []*schemapb.FieldData{&fieldData1, &fieldData2, &fieldData3}
} }
func generateSearchResult() []map[string]interface{} { func generateSearchResult(dataType schemapb.DataType) []map[string]interface{} {
row1 := map[string]interface{}{ row1 := map[string]interface{}{
DefaultPrimaryFieldName: int64(1), DefaultPrimaryFieldName: int64(1),
FieldBookID: int64(1), FieldBookID: int64(1),
@ -218,6 +231,11 @@ func generateSearchResult() []map[string]interface{} {
FieldBookIntro: []float32{0.3, 0.33}, FieldBookIntro: []float32{0.3, 0.33},
HTTPReturnDistance: float32(0.09), HTTPReturnDistance: float32(0.09),
} }
if dataType == schemapb.DataType_String {
row1[DefaultPrimaryFieldName] = "1"
row2[DefaultPrimaryFieldName] = "2"
row3[DefaultPrimaryFieldName] = "3"
}
return []map[string]interface{}{row1, row2, row3} return []map[string]interface{}{row1, row2, row3}
} }
@ -246,9 +264,9 @@ func generateQueryResult64(withDistance bool) []map[string]interface{} {
} }
func TestPrintCollectionDetails(t *testing.T) { func TestPrintCollectionDetails(t *testing.T) {
coll := generateCollectionSchema(false) coll := generateCollectionSchema(schemapb.DataType_Int64, false)
indexes := generateIndexes() indexes := generateIndexes()
assert.Equal(t, printFields(coll.Fields), []gin.H{ assert.Equal(t, []gin.H{
{ {
HTTPReturnFieldName: FieldBookID, HTTPReturnFieldName: FieldBookID,
HTTPReturnFieldType: "Int64", HTTPReturnFieldType: "Int64",
@ -270,23 +288,23 @@ func TestPrintCollectionDetails(t *testing.T) {
HTTPReturnFieldAutoID: false, HTTPReturnFieldAutoID: false,
HTTPReturnDescription: "", HTTPReturnDescription: "",
}, },
}) }, printFields(coll.Fields))
assert.Equal(t, printIndexes(indexes), []gin.H{ assert.Equal(t, []gin.H{
{ {
HTTPReturnIndexName: DefaultIndexName, HTTPReturnIndexName: DefaultIndexName,
HTTPReturnIndexField: FieldBookIntro, HTTPReturnIndexField: FieldBookIntro,
HTTPReturnIndexMetricsType: DefaultMetricType, HTTPReturnIndexMetricsType: DefaultMetricType,
}, },
}) }, printIndexes(indexes))
assert.Equal(t, getMetricType(indexes[0].Params), DefaultMetricType) assert.Equal(t, DefaultMetricType, getMetricType(indexes[0].Params))
assert.Equal(t, getMetricType(nil), DefaultMetricType) assert.Equal(t, DefaultMetricType, getMetricType(nil))
fields := []*schemapb.FieldSchema{} fields := []*schemapb.FieldSchema{}
for _, field := range newCollectionSchema(coll).Fields { for _, field := range newCollectionSchema(coll).Fields {
if field.DataType == schemapb.DataType_VarChar { if field.DataType == schemapb.DataType_VarChar {
fields = append(fields, field) fields = append(fields, field)
} }
} }
assert.Equal(t, printFields(fields), []gin.H{ assert.Equal(t, []gin.H{
{ {
HTTPReturnFieldName: "field-varchar", HTTPReturnFieldName: "field-varchar",
HTTPReturnFieldType: "VarChar(10)", HTTPReturnFieldType: "VarChar(10)",
@ -294,65 +312,65 @@ func TestPrintCollectionDetails(t *testing.T) {
HTTPReturnFieldAutoID: false, HTTPReturnFieldAutoID: false,
HTTPReturnDescription: "", HTTPReturnDescription: "",
}, },
}) }, printFields(fields))
} }
func TestPrimaryField(t *testing.T) { func TestPrimaryField(t *testing.T) {
coll := generateCollectionSchema(false) coll := generateCollectionSchema(schemapb.DataType_Int64, false)
primaryField := generatePrimaryField(schemapb.DataType_Int64) primaryField := generatePrimaryField(schemapb.DataType_Int64)
field, ok := getPrimaryField(coll) field, ok := getPrimaryField(coll)
assert.Equal(t, ok, true) assert.Equal(t, true, ok)
assert.Equal(t, *field, primaryField) assert.Equal(t, primaryField, *field)
assert.Equal(t, joinArray([]int64{1, 2, 3}), "1,2,3") assert.Equal(t, "1,2,3", joinArray([]int64{1, 2, 3}))
assert.Equal(t, joinArray([]string{"1", "2", "3"}), "1,2,3") assert.Equal(t, "1,2,3", joinArray([]string{"1", "2", "3"}))
jsonStr := "{\"id\": [1, 2, 3]}" jsonStr := "{\"id\": [1, 2, 3]}"
idStr := gjson.Get(jsonStr, "id") idStr := gjson.Get(jsonStr, "id")
rangeStr, err := convertRange(&primaryField, idStr) rangeStr, err := convertRange(&primaryField, idStr)
assert.Equal(t, err, nil) assert.Equal(t, nil, err)
assert.Equal(t, rangeStr, "1,2,3") assert.Equal(t, "1,2,3", rangeStr)
filter, err := checkGetPrimaryKey(coll, idStr) filter, err := checkGetPrimaryKey(coll, idStr)
assert.Equal(t, err, nil) assert.Equal(t, nil, err)
assert.Equal(t, filter, "book_id in [1,2,3]") assert.Equal(t, "book_id in [1,2,3]", filter)
primaryField = generatePrimaryField(schemapb.DataType_VarChar) primaryField = generatePrimaryField(schemapb.DataType_VarChar)
jsonStr = "{\"id\": [\"1\", \"2\", \"3\"]}" jsonStr = "{\"id\": [\"1\", \"2\", \"3\"]}"
idStr = gjson.Get(jsonStr, "id") idStr = gjson.Get(jsonStr, "id")
rangeStr, err = convertRange(&primaryField, idStr) rangeStr, err = convertRange(&primaryField, idStr)
assert.Equal(t, err, nil) assert.Equal(t, nil, err)
assert.Equal(t, rangeStr, "1,2,3") assert.Equal(t, "1,2,3", rangeStr)
filter, err = checkGetPrimaryKey(coll, idStr) filter, err = checkGetPrimaryKey(coll, idStr)
assert.Equal(t, err, nil) assert.Equal(t, nil, err)
assert.Equal(t, filter, "book_id in [1,2,3]") assert.Equal(t, "book_id in [1,2,3]", filter)
} }
func TestInsertWithDynamicFields(t *testing.T) { func TestInsertWithDynamicFields(t *testing.T) {
body := "{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}" body := "{\"data\": {\"id\": 0, \"book_id\": 1, \"book_intro\": [0.1, 0.2], \"word_count\": 2, \"classified\": false, \"databaseID\": null}}"
req := InsertReq{} req := InsertReq{}
coll := generateCollectionSchema(false) coll := generateCollectionSchema(schemapb.DataType_Int64, false)
var err error var err error
err, req.Data = checkAndSetData(body, &milvuspb.DescribeCollectionResponse{ err, req.Data = checkAndSetData(body, &milvuspb.DescribeCollectionResponse{
Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success}, Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success},
Schema: coll, Schema: coll,
}) })
assert.Equal(t, err, nil) assert.Equal(t, nil, err)
assert.Equal(t, req.Data[0]["id"], int64(0)) assert.Equal(t, int64(0), req.Data[0]["id"])
assert.Equal(t, req.Data[0]["book_id"], int64(1)) assert.Equal(t, int64(1), req.Data[0]["book_id"])
assert.Equal(t, req.Data[0]["word_count"], int64(2)) assert.Equal(t, int64(2), req.Data[0]["word_count"])
fieldsData, err := anyToColumns(req.Data, coll) fieldsData, err := anyToColumns(req.Data, coll)
assert.Equal(t, err, nil) assert.Equal(t, nil, err)
assert.Equal(t, fieldsData[len(fieldsData)-1].IsDynamic, true) assert.Equal(t, true, fieldsData[len(fieldsData)-1].IsDynamic)
assert.Equal(t, fieldsData[len(fieldsData)-1].Type, schemapb.DataType_JSON) assert.Equal(t, schemapb.DataType_JSON, fieldsData[len(fieldsData)-1].Type)
assert.Equal(t, string(fieldsData[len(fieldsData)-1].GetScalars().GetJsonData().GetData()[0]), "{\"classified\":false,\"id\":0}") assert.Equal(t, "{\"classified\":false,\"id\":0}", string(fieldsData[len(fieldsData)-1].GetScalars().GetJsonData().GetData()[0]))
} }
func TestSerialize(t *testing.T) { func TestSerialize(t *testing.T) {
parameters := []float32{0.11111, 0.22222} parameters := []float32{0.11111, 0.22222}
// assert.Equal(t, string(serialize(parameters)), "\ufffd\ufffd\ufffd=\ufffd\ufffdc\u003e") // assert.Equal(t, "\ufffd\ufffd\ufffd=\ufffd\ufffdc\u003e", string(serialize(parameters)))
// assert.Equal(t, string(vector2PlaceholderGroupBytes(parameters)), "vector2PlaceholderGroupBytes") // todo // assert.Equal(t, "vector2PlaceholderGroupBytes", string(vector2PlaceholderGroupBytes(parameters))) // todo
assert.Equal(t, string(serialize(parameters)), "\xa4\x8d\xe3=\xa4\x8dc>") assert.Equal(t, "\xa4\x8d\xe3=\xa4\x8dc>", string(serialize(parameters)))
assert.Equal(t, string(vector2PlaceholderGroupBytes(parameters)), "\n\x10\n\x02$0\x10e\x1a\b\xa4\x8d\xe3=\xa4\x8dc>") // todo assert.Equal(t, "\n\x10\n\x02$0\x10e\x1a\b\xa4\x8d\xe3=\xa4\x8dc>", string(vector2PlaceholderGroupBytes(parameters))) // todo
} }
func compareRow64(m1 map[string]interface{}, m2 map[string]interface{}) bool { func compareRow64(m1 map[string]interface{}, m2 map[string]interface{}) bool {
@ -438,10 +456,10 @@ func compareRows(row1 []map[string]interface{}, row2 []map[string]interface{}, c
func TestBuildQueryResp(t *testing.T) { func TestBuildQueryResp(t *testing.T) {
outputFields := []string{FieldBookID, FieldWordCount, "author", "date"} outputFields := []string{FieldBookID, FieldWordCount, "author", "date"}
rows, err := buildQueryResp(int64(0), outputFields, generateFieldData(), generateIds(3), []float32{0.01, 0.04, 0.09}) // []*schemapb.FieldData{&fieldData1, &fieldData2, &fieldData3} rows, err := buildQueryResp(int64(0), outputFields, generateFieldData(), generateIds(schemapb.DataType_Int64, 3), []float32{0.01, 0.04, 0.09}, true) // []*schemapb.FieldData{&fieldData1, &fieldData2, &fieldData3}
assert.Equal(t, err, nil) assert.Equal(t, nil, err)
exceptRows := generateSearchResult() exceptRows := generateSearchResult(schemapb.DataType_Int64)
assert.Equal(t, compareRows(rows, exceptRows, compareRow), true) assert.Equal(t, true, compareRows(rows, exceptRows, compareRow))
} }
func newCollectionSchema(coll *schemapb.CollectionSchema) *schemapb.CollectionSchema { func newCollectionSchema(coll *schemapb.CollectionSchema) *schemapb.CollectionSchema {
@ -776,19 +794,19 @@ func newSearchResult(results []map[string]interface{}) []map[string]interface{}
} }
func TestAnyToColumn(t *testing.T) { func TestAnyToColumn(t *testing.T) {
data, err := anyToColumns(newSearchResult(generateSearchResult()), newCollectionSchema(generateCollectionSchema(false))) data, err := anyToColumns(newSearchResult(generateSearchResult(schemapb.DataType_Int64)), newCollectionSchema(generateCollectionSchema(schemapb.DataType_Int64, false)))
assert.Equal(t, err, nil) assert.Equal(t, nil, err)
assert.Equal(t, len(data), 13) assert.Equal(t, 13, len(data))
} }
func TestBuildQueryResps(t *testing.T) { func TestBuildQueryResps(t *testing.T) {
outputFields := []string{"XXX", "YYY"} outputFields := []string{"XXX", "YYY"}
outputFieldsList := [][]string{outputFields, {"$meta"}, {"$meta", FieldBookID, FieldBookIntro, "YYY"}} outputFieldsList := [][]string{outputFields, {"$meta"}, {"$meta", FieldBookID, FieldBookIntro, "YYY"}}
for _, theOutputFields := range outputFieldsList { for _, theOutputFields := range outputFieldsList {
rows, err := buildQueryResp(int64(0), theOutputFields, newFieldData(generateFieldData(), schemapb.DataType_None), generateIds(3), []float32{0.01, 0.04, 0.09}) rows, err := buildQueryResp(int64(0), theOutputFields, newFieldData(generateFieldData(), schemapb.DataType_None), generateIds(schemapb.DataType_Int64, 3), []float32{0.01, 0.04, 0.09}, true)
assert.Equal(t, err, nil) assert.Equal(t, nil, err)
exceptRows := newSearchResult(generateSearchResult()) exceptRows := newSearchResult(generateSearchResult(schemapb.DataType_Int64))
assert.Equal(t, compareRows(rows, exceptRows, compareRow), true) assert.Equal(t, true, compareRows(rows, exceptRows, compareRow))
} }
dataTypes := []schemapb.DataType{ dataTypes := []schemapb.DataType{
@ -799,18 +817,29 @@ func TestBuildQueryResps(t *testing.T) {
schemapb.DataType_JSON, schemapb.DataType_Array, schemapb.DataType_JSON, schemapb.DataType_Array,
} }
for _, dateType := range dataTypes { for _, dateType := range dataTypes {
_, err := buildQueryResp(int64(0), outputFields, newFieldData([]*schemapb.FieldData{}, dateType), generateIds(3), []float32{0.01, 0.04, 0.09}) _, err := buildQueryResp(int64(0), outputFields, newFieldData([]*schemapb.FieldData{}, dateType), generateIds(schemapb.DataType_Int64, 3), []float32{0.01, 0.04, 0.09}, true)
assert.Equal(t, err, nil) assert.Equal(t, nil, err)
} }
_, err := buildQueryResp(int64(0), outputFields, newFieldData([]*schemapb.FieldData{}, 1000), generateIds(3), []float32{0.01, 0.04, 0.09}) _, err := buildQueryResp(int64(0), outputFields, newFieldData([]*schemapb.FieldData{}, 1000), generateIds(schemapb.DataType_Int64, 3), []float32{0.01, 0.04, 0.09}, true)
assert.Equal(t, err.Error(), "the type(1000) of field(wrong-field-type) is not supported, use other sdk please") assert.Equal(t, "the type(1000) of field(wrong-field-type) is not supported, use other sdk please", err.Error())
res, err := buildQueryResp(int64(0), outputFields, []*schemapb.FieldData{}, generateIds(3), []float32{0.01, 0.04, 0.09}) res, err := buildQueryResp(int64(0), outputFields, []*schemapb.FieldData{}, generateIds(schemapb.DataType_Int64, 3), []float32{0.01, 0.04, 0.09}, true)
assert.Equal(t, len(res), 3) assert.Equal(t, 3, len(res))
assert.Equal(t, err, nil) assert.Equal(t, nil, err)
res, err = buildQueryResp(int64(0), outputFields, []*schemapb.FieldData{}, generateIds(schemapb.DataType_Int64, 3), []float32{0.01, 0.04, 0.09}, false)
assert.Equal(t, 3, len(res))
assert.Equal(t, nil, err)
res, err = buildQueryResp(int64(0), outputFields, []*schemapb.FieldData{}, generateIds(schemapb.DataType_VarChar, 3), []float32{0.01, 0.04, 0.09}, true)
assert.Equal(t, 3, len(res))
assert.Equal(t, nil, err)
_, err = buildQueryResp(int64(0), outputFields, generateFieldData(), generateIds(schemapb.DataType_Int64, 3), []float32{0.01, 0.04, 0.09}, false)
assert.Equal(t, nil, err)
// len(rows) != len(scores), didn't show distance // len(rows) != len(scores), didn't show distance
_, err = buildQueryResp(int64(0), outputFields, newFieldData(generateFieldData(), schemapb.DataType_None), generateIds(3), []float32{0.01, 0.04}) _, err = buildQueryResp(int64(0), outputFields, newFieldData(generateFieldData(), schemapb.DataType_None), generateIds(schemapb.DataType_Int64, 3), []float32{0.01, 0.04}, true)
assert.Equal(t, err, nil) assert.Equal(t, nil, err)
} }

9
internal/distributed/proxy/service.go
View File

@ -170,6 +170,15 @@ func (s *Server) startHTTPServer(errChan chan error) {
defer s.wg.Done() defer s.wg.Done()
ginHandler := gin.Default() ginHandler := gin.Default()
ginHandler.Use(func(c *gin.Context) { ginHandler.Use(func(c *gin.Context) {
_, err := strconv.ParseBool(c.Request.Header.Get(httpserver.HTTPHeaderAllowInt64))
if err != nil {
httpParams := &paramtable.Get().HTTPCfg
if httpParams.AcceptTypeAllowInt64.GetAsBool() {
c.Request.Header.Set(httpserver.HTTPHeaderAllowInt64, "true")
} else {
c.Request.Header.Set(httpserver.HTTPHeaderAllowInt64, "false")
}
}
c.Writer.Header().Set("Access-Control-Allow-Origin", "*") c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
c.Writer.Header().Set("Access-Control-Allow-Credentials", "true") c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With") c.Writer.Header().Set("Access-Control-Allow-Headers", "Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, accept, origin, Cache-Control, X-Requested-With")

11
pkg/util/paramtable/http_param.go
View File

@ -4,6 +4,7 @@ type httpConfig struct {
Enabled ParamItem `refreshable:"false"` Enabled ParamItem `refreshable:"false"`
DebugMode ParamItem `refreshable:"false"` DebugMode ParamItem `refreshable:"false"`
Port ParamItem `refreshable:"false"` Port ParamItem `refreshable:"false"`
AcceptTypeAllowInt64 ParamItem `refreshable:"false"`
} }
func (p *httpConfig) init(base *BaseTable) { func (p *httpConfig) init(base *BaseTable) {
@ -33,4 +34,14 @@ func (p *httpConfig) init(base *BaseTable) {
Export: true, Export: true,
} }
p.Port.Init(base.mgr) p.Port.Init(base.mgr)
p.AcceptTypeAllowInt64 = ParamItem{
Key: "proxy.http.acceptTypeAllowInt64",
DefaultValue: "false",
Version: "2.3.2",
Doc: "high-level restful api, whether http client can deal with int64",
PanicIfEmpty: false,
Export: true,
}
p.AcceptTypeAllowInt64.Init(base.mgr)
} }

1
pkg/util/paramtable/http_param_test.go
View File

@ -13,4 +13,5 @@ func TestHTTPConfig_Init(t *testing.T) {
assert.Equal(t, cfg.Enabled.GetAsBool(), true) assert.Equal(t, cfg.Enabled.GetAsBool(), true)
assert.Equal(t, cfg.DebugMode.GetAsBool(), false) assert.Equal(t, cfg.DebugMode.GetAsBool(), false)
assert.Equal(t, cfg.Port.GetValue(), "") assert.Equal(t, cfg.Port.GetValue(), "")
assert.Equal(t, cfg.AcceptTypeAllowInt64.GetValue(), "false")
} }