admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-02-02 01:06:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add reqeust id log for minio chunk manager (#18467)

Browse Source 
Signed-off-by: yun.zhang <yun.zhang@zilliz.com>
This commit is contained in:
jaime 2022-08-01 18:54:33 +08:00 committed by GitHub
parent dcd6c9e5a9
commit f4e7861e6b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
1 changed files with 24 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
internal/storage/minio_chunk_manager.go
View File

@ -75,7 +75,8 @@ func newMinioChunkManagerWithConfig(ctx context.Context, c *config) (*MinioChunk
checkBucketFn := func() error {
bucketExists, err = minIOClient.BucketExists(ctx, c.bucketName)
if err != nil {
log.Warn("failed to check blob bucket exist", zap.String("bucket", c.bucketName), zap.Error(err))
errResponse := minio.ToErrorResponse(err)
log.Warn("failed to check blob bucket exist", zap.String("bucket", c.bucketName), zap.String("requestID", errResponse.RequestID), zap.Error(err))
return err
}
if !bucketExists {
@ -83,7 +84,8 @@ func newMinioChunkManagerWithConfig(ctx context.Context, c *config) (*MinioChunk
log.Info("blob bucket not exist, create bucket.", zap.Any("bucket name", c.bucketName))
err := minIOClient.MakeBucket(ctx, c.bucketName, minio.MakeBucketOptions{})
if err != nil {
log.Warn("failed to create blob bucket", zap.String("bucket", c.bucketName), zap.Error(err))
errResponse := minio.ToErrorResponse(err)
log.Warn("failed to create blob bucket", zap.String("bucket", c.bucketName), zap.String("requestID", errResponse.RequestID), zap.Error(err))
return err
}
} else {
@ -122,7 +124,8 @@ func (mcm *MinioChunkManager) Path(filePath string) (string, error) {
func (mcm *MinioChunkManager) Reader(filePath string) (FileReader, error) {
reader, err := mcm.Client.GetObject(mcm.ctx, mcm.bucketName, filePath, minio.GetObjectOptions{})
if err != nil {
log.Warn("failed to get object", zap.String("path", filePath), zap.Error(err))
errResponse := minio.ToErrorResponse(err)
log.Warn("failed to get object", zap.String("path", filePath), zap.String("requestID", errResponse.RequestID), zap.Error(err))
return nil, err
}
return reader, nil
@ -131,7 +134,8 @@ func (mcm *MinioChunkManager) Reader(filePath string) (FileReader, error) {
func (mcm *MinioChunkManager) Size(filePath string) (int64, error) {
objectInfo, err := mcm.Client.StatObject(mcm.ctx, mcm.bucketName, filePath, minio.StatObjectOptions{})
if err != nil {
log.Warn("failed to stat object", zap.String("path", filePath), zap.Error(err))
errResponse := minio.ToErrorResponse(err)
log.Warn("failed to stat object", zap.String("path", filePath), zap.String("requestID", errResponse.RequestID), zap.Error(err))
return 0, err
}
@ -143,7 +147,8 @@ func (mcm *MinioChunkManager) Write(filePath string, content []byte) error {
_, err := mcm.Client.PutObject(mcm.ctx, mcm.bucketName, filePath, bytes.NewReader(content), int64(len(content)), minio.PutObjectOptions{})
if err != nil {
log.Warn("failed to put object", zap.String("path", filePath), zap.Error(err))
errResponse := minio.ToErrorResponse(err)
log.Warn("failed to put object", zap.String("path", filePath), zap.String("requestID", errResponse.RequestID), zap.Error(err))
return err
}
@ -174,7 +179,7 @@ func (mcm *MinioChunkManager) Exist(filePath string) (bool, error) {
if errResponse.Code == "NoSuchKey" {
return false, nil
}
log.Warn("failed to stat object", zap.String("path", filePath), zap.Error(err))
log.Warn("failed to stat object", zap.String("path", filePath), zap.String("requestID", errResponse.RequestID), zap.Error(err))
return false, err
}
return true, nil
@ -184,7 +189,8 @@ func (mcm *MinioChunkManager) Exist(filePath string) (bool, error) {
func (mcm *MinioChunkManager) Read(filePath string) ([]byte, error) {
object, err := mcm.Client.GetObject(mcm.ctx, mcm.bucketName, filePath, minio.GetObjectOptions{})
if err != nil {
log.Warn("failed to get object", zap.String("path", filePath), zap.Error(err))
errResponse := minio.ToErrorResponse(err)
log.Warn("failed to get object", zap.String("path", filePath), zap.String("requestID", errResponse.RequestID), zap.Error(err))
return nil, err
}
defer object.Close()
@ -195,7 +201,7 @@ func (mcm *MinioChunkManager) Read(filePath string) ([]byte, error) {
if errResponse.Code == "NoSuchKey" {
return nil, errors.New("NoSuchKey")
}
log.Warn("failed to read object", zap.String("path", filePath), zap.Error(err))
log.Warn("failed to read object", zap.String("path", filePath), zap.String("requestID", errResponse.RequestID), zap.Error(err))
return nil, err
}
return data, nil
@ -250,13 +256,15 @@ func (mcm *MinioChunkManager) ReadAt(filePath string, off int64, length int64) (
object, err := mcm.Client.GetObject(mcm.ctx, mcm.bucketName, filePath, opts)
if err != nil {
log.Warn("failed to get object", zap.String("path", filePath), zap.Error(err))
errResponse := minio.ToErrorResponse(err)
log.Warn("failed to get object", zap.String("path", filePath), zap.String("requestID", errResponse.RequestID), zap.Error(err))
return nil, err
}
defer object.Close()
data, err := ioutil.ReadAll(object)
if err != nil {
log.Warn("failed to read object", zap.String("path", filePath), zap.Error(err))
errResponse := minio.ToErrorResponse(err)
log.Warn("failed to read object", zap.String("path", filePath), zap.String("requestID", errResponse.RequestID), zap.Error(err))
return nil, err
}
return data, nil
@ -266,7 +274,8 @@ func (mcm *MinioChunkManager) ReadAt(filePath string, off int64, length int64) (
func (mcm *MinioChunkManager) Remove(filePath string) error {
err := mcm.Client.RemoveObject(mcm.ctx, mcm.bucketName, filePath, minio.RemoveObjectOptions{})
if err != nil {
log.Warn("failed to remove object", zap.String("path", filePath), zap.Error(err))
errResponse := minio.ToErrorResponse(err)
log.Warn("failed to remove object", zap.String("path", filePath), zap.String("requestID", errResponse.RequestID), zap.Error(err))
return err
}
return nil
@ -292,7 +301,8 @@ func (mcm *MinioChunkManager) RemoveWithPrefix(prefix string) error {
objects := mcm.Client.ListObjects(mcm.ctx, mcm.bucketName, minio.ListObjectsOptions{Prefix: prefix, Recursive: true})
for rErr := range mcm.Client.RemoveObjects(mcm.ctx, mcm.bucketName, objects, minio.RemoveObjectsOptions{GovernanceBypass: false}) {
if rErr.Err != nil {
log.Warn("failed to remove objects", zap.String("prefix", prefix), zap.Error(rErr.Err))
errResponse := minio.ToErrorResponse(rErr.Err)
log.Warn("failed to remove objects", zap.String("prefix", prefix), zap.String("requestID", errResponse.RequestID), zap.Error(rErr.Err))
return rErr.Err
}
}
@ -305,7 +315,8 @@ func (mcm *MinioChunkManager) ListWithPrefix(prefix string, recursive bool) ([]s
for object := range objects {
if object.Err != nil {
log.Warn("failed to list with prefix", zap.String("prefix", prefix), zap.Error(object.Err))
errResponse := minio.ToErrorResponse(object.Err)
log.Warn("failed to list with prefix", zap.String("prefix", prefix), zap.String("requestID", errResponse.RequestID), zap.Error(object.Err))
return nil, object.Err
}
objectsKeys = append(objectsKeys, object.Key)