admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-01-07 19:31:51 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix:[CHERRY_PICK] link with install path's libblob-chunk-manager (#29731)

Browse Source 
issue: #29494
pr: #29496


1. link with install path's libblob-chunk-manager
2. performance of `ShouldBindWith` is better than `ShouldBindBodyWith`
3. the middleware shouldn't read the unrefreshed parameter repeatly
4. diff from old restful, named HandlersV1

Signed-off-by: PowderLi <min.li@zilliz.com>
This commit is contained in:
PowderLi 2024-01-07 19:52:49 +08:00 committed by GitHub
parent 58410d8b62
commit 9e7974a2c6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 54 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
internal/core/src/storage/CMakeLists.txt
View File

@ -63,6 +63,7 @@ add_library(milvus_storage SHARED ${STORAGE_FILES})
if (DEFINED AZURE_BUILD_DIR)
target_link_libraries(milvus_storage PUBLIC
"-L${AZURE_BUILD_DIR} -lblob-chunk-manager"
blob-chunk-manager
milvus_common
pthread
${CONAN_LIBS}

2
internal/core/src/storage/azure-blob-storage/test/CMakeLists.txt
View File

@ -5,4 +5,4 @@ project(azure-blob-test)
add_executable(azure-blob-test test_azure_blob_chunk_manager.cpp ../AzureBlobChunkManager.cpp)
find_package(GTest CONFIG REQUIRED)
target_link_libraries(azure-blob-test PRIVATE Azure::azure-identity Azure::azure-storage-blobs GTest::gtest)
target_link_libraries(azure-blob-test PRIVATE Azure::azure-identity Azure::azure-storage-blobs GTest::gtest blob-chunk-manager)

49
internal/distributed/proxy/httpserver/handler_v1.go
View File

@ -16,11 +16,24 @@ import (
"github.com/milvus-io/milvus-proto/go-api/v2/milvuspb"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/internal/proxy"
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/util/merr"
)
// HandlersV1 handles http requests
type HandlersV1 struct {
proxy types.ProxyComponent
}
// NewHandlers creates a new HandlersV1
func NewHandlersV1(proxy types.ProxyComponent) *HandlersV1 {
return &HandlersV1{
proxy: proxy,
}
}
func checkAuthorization(ctx context.Context, c *gin.Context, req interface{}) error {
if proxy.Params.CommonCfg.AuthorizationEnabled.GetAsBool() {
username, ok := c.Get(ContextUsername)
@ -37,7 +50,7 @@ func checkAuthorization(ctx context.Context, c *gin.Context, req interface{}) er
return nil
}
func (h *Handlers) checkDatabase(ctx context.Context, c *gin.Context, dbName string) bool {
func (h *HandlersV1) checkDatabase(ctx context.Context, c *gin.Context, dbName string) bool {
if dbName == DefaultDbName {
return true
}
@ -64,7 +77,7 @@ func (h *Handlers) checkDatabase(ctx context.Context, c *gin.Context, dbName str
return false
}
func (h *Handlers) describeCollection(ctx context.Context, c *gin.Context, dbName string, collectionName string) (*schemapb.CollectionSchema, error) {
func (h *HandlersV1) describeCollection(ctx context.Context, c *gin.Context, dbName string, collectionName string) (*schemapb.CollectionSchema, error) {
collSchema, err := proxy.GetCachedCollectionSchema(ctx, dbName, collectionName)
if err == nil {
return collSchema, nil
@ -89,7 +102,7 @@ func (h *Handlers) describeCollection(ctx context.Context, c *gin.Context, dbNam
return response.Schema, nil
}
func (h *Handlers) hasCollection(ctx context.Context, c *gin.Context, dbName string, collectionName string) (bool, error) {
func (h *HandlersV1) hasCollection(ctx context.Context, c *gin.Context, dbName string, collectionName string) (bool, error) {
req := milvuspb.HasCollectionRequest{
DbName: dbName,
CollectionName: collectionName,
@ -105,7 +118,7 @@ func (h *Handlers) hasCollection(ctx context.Context, c *gin.Context, dbName str
return response.Value, nil
}
func (h *Handlers) RegisterRoutesToV1(router gin.IRouter) {
func (h *HandlersV1) RegisterRoutesToV1(router gin.IRouter) {
router.GET(VectorCollectionsPath, h.listCollections)
router.POST(VectorCollectionsCreatePath, h.createCollection)
router.GET(VectorCollectionsDescribePath, h.getCollectionDetails)
@ -118,7 +131,7 @@ func (h *Handlers) RegisterRoutesToV1(router gin.IRouter) {
router.POST(VectorSearchPath, h.search)
}
func (h *Handlers) listCollections(c *gin.Context) {
func (h *HandlersV1) listCollections(c *gin.Context) {
dbName := c.DefaultQuery(HTTPDbName, DefaultDbName)
req := milvuspb.ShowCollectionsRequest{
DbName: dbName,
@ -148,14 +161,14 @@ func (h *Handlers) listCollections(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{HTTPReturnCode: http.StatusOK, HTTPReturnData: collections})
}
func (h *Handlers) createCollection(c *gin.Context) {
func (h *HandlersV1) createCollection(c *gin.Context) {
httpReq := CreateCollectionReq{
DbName: DefaultDbName,
MetricType: DefaultMetricType,
PrimaryField: DefaultPrimaryFieldName,
VectorField: DefaultVectorFieldName,
}
if err := c.ShouldBindBodyWith(&httpReq, binding.JSON); err != nil {
if err := c.ShouldBindWith(&httpReq, binding.JSON); err != nil {
log.Warn("high level restful api, the parameter of create collection is incorrect", zap.Any("request", httpReq), zap.Error(err))
c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrIncorrectParameterFormat),
@ -258,7 +271,7 @@ func (h *Handlers) createCollection(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{HTTPReturnCode: http.StatusOK, HTTPReturnData: gin.H{}})
}
func (h *Handlers) getCollectionDetails(c *gin.Context) {
func (h *HandlersV1) getCollectionDetails(c *gin.Context) {
collectionName := c.Query(HTTPCollectionName)
if collectionName == "" {
log.Warn("high level restful api, desc collection require parameter: [collectionName], but miss")
@ -342,11 +355,11 @@ func (h *Handlers) getCollectionDetails(c *gin.Context) {
}})
}
func (h *Handlers) dropCollection(c *gin.Context) {
func (h *HandlersV1) dropCollection(c *gin.Context) {
httpReq := DropCollectionReq{
DbName: DefaultDbName,
}
if err := c.ShouldBindBodyWith(&httpReq, binding.JSON); err != nil {
if err := c.ShouldBindWith(&httpReq, binding.JSON); err != nil {
log.Warn("high level restful api, the parameter of drop collection is incorrect", zap.Any("request", httpReq), zap.Error(err))
c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrIncorrectParameterFormat),
@ -396,13 +409,13 @@ func (h *Handlers) dropCollection(c *gin.Context) {
}
}
func (h *Handlers) query(c *gin.Context) {
func (h *HandlersV1) query(c *gin.Context) {
httpReq := QueryReq{
DbName: DefaultDbName,
Limit: 100,
OutputFields: []string{DefaultOutputFields},
}
if err := c.ShouldBindBodyWith(&httpReq, binding.JSON); err != nil {
if err := c.ShouldBindWith(&httpReq, binding.JSON); err != nil {
log.Warn("high level restful api, the parameter of query is incorrect", zap.Any("request", httpReq), zap.Error(err))
c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrIncorrectParameterFormat),
@ -461,7 +474,7 @@ func (h *Handlers) query(c *gin.Context) {
}
}
func (h *Handlers) get(c *gin.Context) {
func (h *HandlersV1) get(c *gin.Context) {
httpReq := GetReq{
DbName: DefaultDbName,
OutputFields: []string{DefaultOutputFields},
@ -531,7 +544,7 @@ func (h *Handlers) get(c *gin.Context) {
}
}
func (h *Handlers) delete(c *gin.Context) {
func (h *HandlersV1) delete(c *gin.Context) {
httpReq := DeleteReq{
DbName: DefaultDbName,
}
@ -591,7 +604,7 @@ func (h *Handlers) delete(c *gin.Context) {
}
}
func (h *Handlers) insert(c *gin.Context) {
func (h *HandlersV1) insert(c *gin.Context) {
httpReq := InsertReq{
DbName: DefaultDbName,
}
@ -682,7 +695,7 @@ func (h *Handlers) insert(c *gin.Context) {
}
}
func (h *Handlers) upsert(c *gin.Context) {
func (h *HandlersV1) upsert(c *gin.Context) {
httpReq := UpsertReq{
DbName: DefaultDbName,
}
@ -778,12 +791,12 @@ func (h *Handlers) upsert(c *gin.Context) {
}
}
func (h *Handlers) search(c *gin.Context) {
func (h *HandlersV1) search(c *gin.Context) {
httpReq := SearchReq{
DbName: DefaultDbName,
Limit: 100,
}
if err := c.ShouldBindBodyWith(&httpReq, binding.JSON); err != nil {
if err := c.ShouldBindWith(&httpReq, binding.JSON); err != nil {
log.Warn("high level restful api, the parameter of search is incorrect", zap.Any("request", httpReq), zap.Error(err))
c.AbortWithStatusJSON(http.StatusOK, gin.H{
HTTPReturnCode: merr.Code(merr.ErrIncorrectParameterFormat),

3
internal/distributed/proxy/httpserver/handler_v1_test.go
View File

@ -83,7 +83,6 @@ func versional(path string) string {
}
func initHTTPServer(proxy types.ProxyComponent, needAuth bool) *gin.Engine {
h := NewHandlers(proxy)
ginHandler := gin.Default()
ginHandler.Use(func(c *gin.Context) {
_, err := strconv.ParseBool(c.Request.Header.Get(HTTPHeaderAllowInt64))
@ -98,7 +97,7 @@ func initHTTPServer(proxy types.ProxyComponent, needAuth bool) *gin.Engine {
c.Next()
})
app := ginHandler.Group(URIPrefixV1, genAuthMiddleWare(needAuth))
NewHandlers(h.proxy).RegisterRoutesToV1(app)
NewHandlersV1(proxy).RegisterRoutesToV1(app)
return ginHandler
}

26
internal/distributed/proxy/service.go
View File

@ -124,10 +124,6 @@ func NewServer(ctx context.Context, factory dependency.Factory) (*Server, error)
}
func authenticate(c *gin.Context) {
c.Set(httpserver.ContextUsername, "")
if !proxy.Params.CommonCfg.AuthorizationEnabled.GetAsBool() {
return
}
// TODO fubang
username, password, ok := httpserver.ParseUsernamePassword(c)
if ok {
@ -178,15 +174,15 @@ func (s *Server) startHTTPServer(errChan chan error) {
SkipPaths: proxy.Params.ProxyCfg.GinLogSkipPaths.GetAsStrings(),
})
ginHandler.Use(ginLogger, gin.Recovery())
httpHeaderAllowInt64 := "false"
httpParams := &paramtable.Get().HTTPCfg
if httpParams.AcceptTypeAllowInt64.GetAsBool() {
httpHeaderAllowInt64 = "true"
}
ginHandler.Use(func(c *gin.Context) {
_, err := strconv.ParseBool(c.Request.Header.Get(httpserver.HTTPHeaderAllowInt64))
if err != nil {
httpParams := &paramtable.Get().HTTPCfg
if httpParams.AcceptTypeAllowInt64.GetAsBool() {
c.Request.Header.Set(httpserver.HTTPHeaderAllowInt64, "true")
} else {
c.Request.Header.Set(httpserver.HTTPHeaderAllowInt64, "false")
}
c.Request.Header.Set(httpserver.HTTPHeaderAllowInt64, httpHeaderAllowInt64)
}
c.Writer.Header().Set("Access-Control-Allow-Origin", "*")
c.Writer.Header().Set("Access-Control-Allow-Credentials", "true")
@ -198,8 +194,14 @@ func (s *Server) startHTTPServer(errChan chan error) {
}
c.Next()
})
app := ginHandler.Group("/v1", authenticate)
httpserver.NewHandlers(s.proxy).RegisterRoutesToV1(app)
ginHandler.Use(func(c *gin.Context) {
c.Set(httpserver.ContextUsername, "")
})
if proxy.Params.CommonCfg.AuthorizationEnabled.GetAsBool() {
ginHandler.Use(authenticate)
}
app := ginHandler.Group("/v1")
httpserver.NewHandlersV1(s.proxy).RegisterRoutesToV1(app)
s.httpServer = &http.Server{Handler: ginHandler, ReadHeaderTimeout: time.Second}
errChan <- nil
if err := s.httpServer.Serve(s.httpListener); err != nil && err != cmux.ErrServerClosed {

2
scripts/azure_build.sh
View File

@ -50,4 +50,4 @@ fi
echo ${AZURE_CMAKE_CMD}
${AZURE_CMAKE_CMD}
make & make install
make install

5
scripts/core_build.sh
View File

@ -186,9 +186,14 @@ if [ -z "$BUILD_WITHOUT_AZURE" ]; then
pushd ${AZURE_BUILD_DIR}
env bash ${ROOT_DIR}/scripts/azure_build.sh -p ${INSTALL_PREFIX} -s ${ROOT_DIR}/internal/core/src/storage/azure-blob-storage -t ${BUILD_UNITTEST}
if [ ! -e libblob-chunk-manager* ]; then
echo "build blob-chunk-manager fail..."
cat vcpkg-bootstrap.log
exit 1
fi
if [ ! -e ${INSTALL_PREFIX}/lib/libblob-chunk-manager* ]; then
echo "install blob-chunk-manager fail..."
exit 1
fi
popd
SYSTEM_NAME=$(uname -s)
if [[ ${SYSTEM_NAME} == "Darwin" ]]; then