diff --git a/internal/core/src/storage/CMakeLists.txt b/internal/core/src/storage/CMakeLists.txt index a21a80d455..d134a4f61e 100644 --- a/internal/core/src/storage/CMakeLists.txt +++ b/internal/core/src/storage/CMakeLists.txt @@ -63,6 +63,7 @@ add_library(milvus_storage SHARED ${STORAGE_FILES}) if (DEFINED AZURE_BUILD_DIR) target_link_libraries(milvus_storage PUBLIC "-L${AZURE_BUILD_DIR} -lblob-chunk-manager" + blob-chunk-manager milvus_common pthread ${CONAN_LIBS} diff --git a/internal/core/src/storage/azure-blob-storage/test/CMakeLists.txt b/internal/core/src/storage/azure-blob-storage/test/CMakeLists.txt index 3dbbeb0b84..563fcfd6a7 100644 --- a/internal/core/src/storage/azure-blob-storage/test/CMakeLists.txt +++ b/internal/core/src/storage/azure-blob-storage/test/CMakeLists.txt @@ -5,4 +5,4 @@ project(azure-blob-test) add_executable(azure-blob-test test_azure_blob_chunk_manager.cpp ../AzureBlobChunkManager.cpp) find_package(GTest CONFIG REQUIRED) -target_link_libraries(azure-blob-test PRIVATE Azure::azure-identity Azure::azure-storage-blobs GTest::gtest) +target_link_libraries(azure-blob-test PRIVATE Azure::azure-identity Azure::azure-storage-blobs GTest::gtest blob-chunk-manager) diff --git a/internal/distributed/proxy/httpserver/handler_v1.go b/internal/distributed/proxy/httpserver/handler_v1.go index 6f011584c4..e8c7b8d13a 100644 --- a/internal/distributed/proxy/httpserver/handler_v1.go +++ b/internal/distributed/proxy/httpserver/handler_v1.go @@ -16,11 +16,24 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/internal/proxy" + "github.com/milvus-io/milvus/internal/types" "github.com/milvus-io/milvus/pkg/common" "github.com/milvus-io/milvus/pkg/log" "github.com/milvus-io/milvus/pkg/util/merr" ) +// HandlersV1 handles http requests +type HandlersV1 struct { + proxy types.ProxyComponent +} + +// NewHandlers creates a new HandlersV1 +func NewHandlersV1(proxy types.ProxyComponent) *HandlersV1 { + return &HandlersV1{ + proxy: proxy, + } +} + func checkAuthorization(ctx context.Context, c *gin.Context, req interface{}) error { if proxy.Params.CommonCfg.AuthorizationEnabled.GetAsBool() { username, ok := c.Get(ContextUsername) @@ -37,7 +50,7 @@ func checkAuthorization(ctx context.Context, c *gin.Context, req interface{}) er return nil } -func (h *Handlers) checkDatabase(ctx context.Context, c *gin.Context, dbName string) bool { +func (h *HandlersV1) checkDatabase(ctx context.Context, c *gin.Context, dbName string) bool { if dbName == DefaultDbName { return true } @@ -64,7 +77,7 @@ func (h *Handlers) checkDatabase(ctx context.Context, c *gin.Context, dbName str return false } -func (h *Handlers) describeCollection(ctx context.Context, c *gin.Context, dbName string, collectionName string) (*schemapb.CollectionSchema, error) { +func (h *HandlersV1) describeCollection(ctx context.Context, c *gin.Context, dbName string, collectionName string) (*schemapb.CollectionSchema, error) { collSchema, err := proxy.GetCachedCollectionSchema(ctx, dbName, collectionName) if err == nil { return collSchema, nil @@ -89,7 +102,7 @@ func (h *Handlers) describeCollection(ctx context.Context, c *gin.Context, dbNam return response.Schema, nil } -func (h *Handlers) hasCollection(ctx context.Context, c *gin.Context, dbName string, collectionName string) (bool, error) { +func (h *HandlersV1) hasCollection(ctx context.Context, c *gin.Context, dbName string, collectionName string) (bool, error) { req := milvuspb.HasCollectionRequest{ DbName: dbName, CollectionName: collectionName, @@ -105,7 +118,7 @@ func (h *Handlers) hasCollection(ctx context.Context, c *gin.Context, dbName str return response.Value, nil } -func (h *Handlers) RegisterRoutesToV1(router gin.IRouter) { +func (h *HandlersV1) RegisterRoutesToV1(router gin.IRouter) { router.GET(VectorCollectionsPath, h.listCollections) router.POST(VectorCollectionsCreatePath, h.createCollection) router.GET(VectorCollectionsDescribePath, h.getCollectionDetails) @@ -118,7 +131,7 @@ func (h *Handlers) RegisterRoutesToV1(router gin.IRouter) { router.POST(VectorSearchPath, h.search) } -func (h *Handlers) listCollections(c *gin.Context) { +func (h *HandlersV1) listCollections(c *gin.Context) { dbName := c.DefaultQuery(HTTPDbName, DefaultDbName) req := milvuspb.ShowCollectionsRequest{ DbName: dbName, @@ -148,14 +161,14 @@ func (h *Handlers) listCollections(c *gin.Context) { c.JSON(http.StatusOK, gin.H{HTTPReturnCode: http.StatusOK, HTTPReturnData: collections}) } -func (h *Handlers) createCollection(c *gin.Context) { +func (h *HandlersV1) createCollection(c *gin.Context) { httpReq := CreateCollectionReq{ DbName: DefaultDbName, MetricType: DefaultMetricType, PrimaryField: DefaultPrimaryFieldName, VectorField: DefaultVectorFieldName, } - if err := c.ShouldBindBodyWith(&httpReq, binding.JSON); err != nil { + if err := c.ShouldBindWith(&httpReq, binding.JSON); err != nil { log.Warn("high level restful api, the parameter of create collection is incorrect", zap.Any("request", httpReq), zap.Error(err)) c.AbortWithStatusJSON(http.StatusOK, gin.H{ HTTPReturnCode: merr.Code(merr.ErrIncorrectParameterFormat), @@ -258,7 +271,7 @@ func (h *Handlers) createCollection(c *gin.Context) { c.JSON(http.StatusOK, gin.H{HTTPReturnCode: http.StatusOK, HTTPReturnData: gin.H{}}) } -func (h *Handlers) getCollectionDetails(c *gin.Context) { +func (h *HandlersV1) getCollectionDetails(c *gin.Context) { collectionName := c.Query(HTTPCollectionName) if collectionName == "" { log.Warn("high level restful api, desc collection require parameter: [collectionName], but miss") @@ -342,11 +355,11 @@ func (h *Handlers) getCollectionDetails(c *gin.Context) { }}) } -func (h *Handlers) dropCollection(c *gin.Context) { +func (h *HandlersV1) dropCollection(c *gin.Context) { httpReq := DropCollectionReq{ DbName: DefaultDbName, } - if err := c.ShouldBindBodyWith(&httpReq, binding.JSON); err != nil { + if err := c.ShouldBindWith(&httpReq, binding.JSON); err != nil { log.Warn("high level restful api, the parameter of drop collection is incorrect", zap.Any("request", httpReq), zap.Error(err)) c.AbortWithStatusJSON(http.StatusOK, gin.H{ HTTPReturnCode: merr.Code(merr.ErrIncorrectParameterFormat), @@ -396,13 +409,13 @@ func (h *Handlers) dropCollection(c *gin.Context) { } } -func (h *Handlers) query(c *gin.Context) { +func (h *HandlersV1) query(c *gin.Context) { httpReq := QueryReq{ DbName: DefaultDbName, Limit: 100, OutputFields: []string{DefaultOutputFields}, } - if err := c.ShouldBindBodyWith(&httpReq, binding.JSON); err != nil { + if err := c.ShouldBindWith(&httpReq, binding.JSON); err != nil { log.Warn("high level restful api, the parameter of query is incorrect", zap.Any("request", httpReq), zap.Error(err)) c.AbortWithStatusJSON(http.StatusOK, gin.H{ HTTPReturnCode: merr.Code(merr.ErrIncorrectParameterFormat), @@ -461,7 +474,7 @@ func (h *Handlers) query(c *gin.Context) { } } -func (h *Handlers) get(c *gin.Context) { +func (h *HandlersV1) get(c *gin.Context) { httpReq := GetReq{ DbName: DefaultDbName, OutputFields: []string{DefaultOutputFields}, @@ -531,7 +544,7 @@ func (h *Handlers) get(c *gin.Context) { } } -func (h *Handlers) delete(c *gin.Context) { +func (h *HandlersV1) delete(c *gin.Context) { httpReq := DeleteReq{ DbName: DefaultDbName, } @@ -591,7 +604,7 @@ func (h *Handlers) delete(c *gin.Context) { } } -func (h *Handlers) insert(c *gin.Context) { +func (h *HandlersV1) insert(c *gin.Context) { httpReq := InsertReq{ DbName: DefaultDbName, } @@ -682,7 +695,7 @@ func (h *Handlers) insert(c *gin.Context) { } } -func (h *Handlers) upsert(c *gin.Context) { +func (h *HandlersV1) upsert(c *gin.Context) { httpReq := UpsertReq{ DbName: DefaultDbName, } @@ -778,12 +791,12 @@ func (h *Handlers) upsert(c *gin.Context) { } } -func (h *Handlers) search(c *gin.Context) { +func (h *HandlersV1) search(c *gin.Context) { httpReq := SearchReq{ DbName: DefaultDbName, Limit: 100, } - if err := c.ShouldBindBodyWith(&httpReq, binding.JSON); err != nil { + if err := c.ShouldBindWith(&httpReq, binding.JSON); err != nil { log.Warn("high level restful api, the parameter of search is incorrect", zap.Any("request", httpReq), zap.Error(err)) c.AbortWithStatusJSON(http.StatusOK, gin.H{ HTTPReturnCode: merr.Code(merr.ErrIncorrectParameterFormat), diff --git a/internal/distributed/proxy/httpserver/handler_v1_test.go b/internal/distributed/proxy/httpserver/handler_v1_test.go index de3acf5551..544d145328 100644 --- a/internal/distributed/proxy/httpserver/handler_v1_test.go +++ b/internal/distributed/proxy/httpserver/handler_v1_test.go @@ -83,7 +83,6 @@ func versional(path string) string { } func initHTTPServer(proxy types.ProxyComponent, needAuth bool) *gin.Engine { - h := NewHandlers(proxy) ginHandler := gin.Default() ginHandler.Use(func(c *gin.Context) { _, err := strconv.ParseBool(c.Request.Header.Get(HTTPHeaderAllowInt64)) @@ -98,7 +97,7 @@ func initHTTPServer(proxy types.ProxyComponent, needAuth bool) *gin.Engine { c.Next() }) app := ginHandler.Group(URIPrefixV1, genAuthMiddleWare(needAuth)) - NewHandlers(h.proxy).RegisterRoutesToV1(app) + NewHandlersV1(proxy).RegisterRoutesToV1(app) return ginHandler } diff --git a/internal/distributed/proxy/service.go b/internal/distributed/proxy/service.go index 7a3a768f58..0ff4fe7c21 100644 --- a/internal/distributed/proxy/service.go +++ b/internal/distributed/proxy/service.go @@ -124,10 +124,6 @@ func NewServer(ctx context.Context, factory dependency.Factory) (*Server, error) } func authenticate(c *gin.Context) { - c.Set(httpserver.ContextUsername, "") - if !proxy.Params.CommonCfg.AuthorizationEnabled.GetAsBool() { - return - } // TODO fubang username, password, ok := httpserver.ParseUsernamePassword(c) if ok { @@ -178,15 +174,15 @@ func (s *Server) startHTTPServer(errChan chan error) { SkipPaths: proxy.Params.ProxyCfg.GinLogSkipPaths.GetAsStrings(), }) ginHandler.Use(ginLogger, gin.Recovery()) + httpHeaderAllowInt64 := "false" + httpParams := ¶mtable.Get().HTTPCfg + if httpParams.AcceptTypeAllowInt64.GetAsBool() { + httpHeaderAllowInt64 = "true" + } ginHandler.Use(func(c *gin.Context) { _, err := strconv.ParseBool(c.Request.Header.Get(httpserver.HTTPHeaderAllowInt64)) if err != nil { - httpParams := ¶mtable.Get().HTTPCfg - if httpParams.AcceptTypeAllowInt64.GetAsBool() { - c.Request.Header.Set(httpserver.HTTPHeaderAllowInt64, "true") - } else { - c.Request.Header.Set(httpserver.HTTPHeaderAllowInt64, "false") - } + c.Request.Header.Set(httpserver.HTTPHeaderAllowInt64, httpHeaderAllowInt64) } c.Writer.Header().Set("Access-Control-Allow-Origin", "*") c.Writer.Header().Set("Access-Control-Allow-Credentials", "true") @@ -198,8 +194,14 @@ func (s *Server) startHTTPServer(errChan chan error) { } c.Next() }) - app := ginHandler.Group("/v1", authenticate) - httpserver.NewHandlers(s.proxy).RegisterRoutesToV1(app) + ginHandler.Use(func(c *gin.Context) { + c.Set(httpserver.ContextUsername, "") + }) + if proxy.Params.CommonCfg.AuthorizationEnabled.GetAsBool() { + ginHandler.Use(authenticate) + } + app := ginHandler.Group("/v1") + httpserver.NewHandlersV1(s.proxy).RegisterRoutesToV1(app) s.httpServer = &http.Server{Handler: ginHandler, ReadHeaderTimeout: time.Second} errChan <- nil if err := s.httpServer.Serve(s.httpListener); err != nil && err != cmux.ErrServerClosed { diff --git a/scripts/azure_build.sh b/scripts/azure_build.sh index 60d2717feb..0ea585eb36 100644 --- a/scripts/azure_build.sh +++ b/scripts/azure_build.sh @@ -50,4 +50,4 @@ fi echo ${AZURE_CMAKE_CMD} ${AZURE_CMAKE_CMD} -make & make install \ No newline at end of file +make install \ No newline at end of file diff --git a/scripts/core_build.sh b/scripts/core_build.sh index b65816b65d..bd24d116e8 100755 --- a/scripts/core_build.sh +++ b/scripts/core_build.sh @@ -186,9 +186,14 @@ if [ -z "$BUILD_WITHOUT_AZURE" ]; then pushd ${AZURE_BUILD_DIR} env bash ${ROOT_DIR}/scripts/azure_build.sh -p ${INSTALL_PREFIX} -s ${ROOT_DIR}/internal/core/src/storage/azure-blob-storage -t ${BUILD_UNITTEST} if [ ! -e libblob-chunk-manager* ]; then + echo "build blob-chunk-manager fail..." cat vcpkg-bootstrap.log exit 1 fi + if [ ! -e ${INSTALL_PREFIX}/lib/libblob-chunk-manager* ]; then + echo "install blob-chunk-manager fail..." + exit 1 + fi popd SYSTEM_NAME=$(uname -s) if [[ ${SYSTEM_NAME} == "Darwin" ]]; then