admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-08 01:58:34 +08:00
Code Issues Packages Projects Releases Wiki Activity

enhance: Add import option to skip disk quota check (#35274)

Browse Source 
Add an option to skip the disk quota check for backup-restore import.

issue: https://github.com/milvus-io/milvus/issues/33775

Signed-off-by: bigsheeper <yihao.dai@zilliz.com>
This commit is contained in:
yihao.dai 2024-08-05 16:40:16 +08:00 committed by GitHub
parent 6542c1ab0e
commit b71e058bc5
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 27 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/datacoord/import_util.go
View File

@ -272,6 +272,10 @@ func CheckDiskQuota(job ImportJob, meta *meta, imeta ImportMeta) (int64, error)
if !Params.QuotaConfig.DiskProtectionEnabled.GetAsBool() { if !Params.QuotaConfig.DiskProtectionEnabled.GetAsBool() {
return 0, nil return 0, nil
} }
if importutilv2.SkipDiskQuotaCheck(job.GetOptions()) {
log.Info("skip disk quota check for import", zap.Int64("jobID", job.GetJobID()))
return 0, nil
}
var ( var (
requestedTotal int64 requestedTotal int64

9
internal/datacoord/import_util_test.go
View File

@ -35,6 +35,7 @@ import (
"github.com/milvus-io/milvus/internal/proto/datapb" "github.com/milvus-io/milvus/internal/proto/datapb"
"github.com/milvus-io/milvus/internal/proto/internalpb" "github.com/milvus-io/milvus/internal/proto/internalpb"
"github.com/milvus-io/milvus/internal/storage" "github.com/milvus-io/milvus/internal/storage"
"github.com/milvus-io/milvus/internal/util/importutilv2"
"github.com/milvus-io/milvus/pkg/util/merr" "github.com/milvus-io/milvus/pkg/util/merr"
"github.com/milvus-io/milvus/pkg/util/paramtable" "github.com/milvus-io/milvus/pkg/util/paramtable"
) )
@ -280,6 +281,14 @@ func TestImportUtil_CheckDiskQuota(t *testing.T) {
assert.NoError(t, err) assert.NoError(t, err)
Params.Save(Params.QuotaConfig.DiskProtectionEnabled.Key, "true") Params.Save(Params.QuotaConfig.DiskProtectionEnabled.Key, "true")
job.Options = []*commonpb.KeyValuePair{
{Key: importutilv2.BackupFlag, Value: "true"},
{Key: importutilv2.SkipDQC, Value: "true"},
}
_, err = CheckDiskQuota(job, meta, imeta)
assert.NoError(t, err)
job.Options = nil
Params.Save(Params.QuotaConfig.DiskQuota.Key, "10000") Params.Save(Params.QuotaConfig.DiskQuota.Key, "10000")
Params.Save(Params.QuotaConfig.DiskQuotaPerCollection.Key, "10000") Params.Save(Params.QuotaConfig.DiskQuotaPerCollection.Key, "10000")
defer Params.Reset(Params.QuotaConfig.DiskQuota.Key) defer Params.Reset(Params.QuotaConfig.DiskQuota.Key)

14
internal/util/importutilv2/option.go
View File

@ -35,6 +35,7 @@ const (
EndTs2 = "endTs" EndTs2 = "endTs"
BackupFlag = "backup" BackupFlag = "backup"
L0Import = "l0_import" L0Import = "l0_import"
SkipDQC = "skip_disk_quota_check"
) )
type Options []*commonpb.KeyValuePair type Options []*commonpb.KeyValuePair
@ -85,3 +86,16 @@ func IsL0Import(options Options) bool {
} }
return true return true
} }
// SkipDiskQuotaCheck indicates whether the import skips the disk quota check.
// This option should only be enabled during backup restoration.
func SkipDiskQuotaCheck(options Options) bool {
if !IsBackup(options) {
return false
}
skip, err := funcutil.GetAttrByKeyFromRepeatedKV(SkipDQC, options)
if err != nil || strings.ToLower(skip) != "true" {
return false
}
return true
}