fix: alter collection failed with MMAP setting for STRUCT (#45173)

issue: https://github.com/milvus-io/milvus/issues/45001
ref: https://github.com/milvus-io/milvus/issues/42148

---------

Signed-off-by: SpadeA <tangchenjie1210@gmail.com>
Signed-off-by: aoiasd <zhicheng.yue@zilliz.com>
Signed-off-by: SpadeA-Tang <tangchenjie1210@gmail.com>
Co-authored-by: aoiasd <zhicheng.yue@zilliz.com>
This commit is contained in:
Spade A 2025-11-04 20:19:33 +08:00 committed by GitHub
parent 19af1903f6
commit c0029b788d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 65 additions and 8 deletions

View File

@ -1,7 +1,9 @@
package model package model
import ( import (
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
"github.com/milvus-io/milvus/pkg/v2/common"
) )
type StructArrayField struct { type StructArrayField struct {
@ -9,6 +11,7 @@ type StructArrayField struct {
Name string Name string
Description string Description string
Fields []*Field Fields []*Field
TypeParams []*commonpb.KeyValuePair
} }
func (s *StructArrayField) Clone() *StructArrayField { func (s *StructArrayField) Clone() *StructArrayField {
@ -17,6 +20,7 @@ func (s *StructArrayField) Clone() *StructArrayField {
Name: s.Name, Name: s.Name,
Description: s.Description, Description: s.Description,
Fields: CloneFields(s.Fields), Fields: CloneFields(s.Fields),
TypeParams: common.CloneKeyValuePairs(s.TypeParams),
} }
} }
@ -29,10 +33,12 @@ func CloneStructArrayFields(structArrayFields []*StructArrayField) []*StructArra
} }
func (s *StructArrayField) Equal(other StructArrayField) bool { func (s *StructArrayField) Equal(other StructArrayField) bool {
var paramsA common.KeyValuePairs = s.TypeParams
return s.FieldID == other.FieldID && return s.FieldID == other.FieldID &&
s.Name == other.Name && s.Name == other.Name &&
s.Description == other.Description && s.Description == other.Description &&
CheckFieldsEqual(s.Fields, other.Fields) CheckFieldsEqual(s.Fields, other.Fields) &&
paramsA.Equal(other.TypeParams)
} }
func CheckStructArrayFieldsEqual(structArrayFieldsA, structArrayFieldsB []*StructArrayField) bool { func CheckStructArrayFieldsEqual(structArrayFieldsA, structArrayFieldsB []*StructArrayField) bool {
@ -63,6 +69,7 @@ func MarshalStructArrayFieldModel(structArrayField *StructArrayField) *schemapb.
Name: structArrayField.Name, Name: structArrayField.Name,
Description: structArrayField.Description, Description: structArrayField.Description,
Fields: MarshalFieldModels(structArrayField.Fields), Fields: MarshalFieldModels(structArrayField.Fields),
TypeParams: structArrayField.TypeParams,
} }
} }
@ -88,6 +95,7 @@ func UnmarshalStructArrayFieldModel(fieldSchema *schemapb.StructArrayFieldSchema
Name: fieldSchema.Name, Name: fieldSchema.Name,
Description: fieldSchema.Description, Description: fieldSchema.Description,
Fields: UnmarshalFieldModels(fieldSchema.Fields), Fields: UnmarshalFieldModels(fieldSchema.Fields),
TypeParams: fieldSchema.TypeParams,
} }
} }

View File

@ -285,9 +285,35 @@ func applyCollectionMmapSetting(schema *schemapb.CollectionSchema,
} }
} }
for _, structField := range schema.GetStructArrayFields() { for _, structField := range schema.GetStructArrayFields() {
structTypeParams := structField.GetTypeParams()
structMmapEnabled, structExist := common.IsMmapDataEnabled(structTypeParams...)
// If struct field itself doesn't have mmap setting, inherit from collection
if !structExist && exist &&
!common.FieldHasMmapKey(schema, structField.GetFieldID()) {
structField.TypeParams = append(structField.TypeParams, &commonpb.KeyValuePair{
Key: common.MmapEnabledKey,
Value: strconv.FormatBool(collectionMmapEnabled),
})
// Update struct's mmap state since we just set it
structMmapEnabled = collectionMmapEnabled
structExist = true
}
// Apply mmap setting to fields inside struct
for _, field := range structField.GetFields() { for _, field := range structField.GetFields() {
if exist && // Skip if field already has mmap setting
!common.FieldHasMmapKey(schema, field.GetFieldID()) { if common.FieldHasMmapKey(schema, field.GetFieldID()) {
continue
}
// Priority: struct field setting > collection setting
if structExist {
field.TypeParams = append(field.TypeParams, &commonpb.KeyValuePair{
Key: common.MmapEnabledKey,
Value: strconv.FormatBool(structMmapEnabled),
})
} else if exist {
field.TypeParams = append(field.TypeParams, &commonpb.KeyValuePair{ field.TypeParams = append(field.TypeParams, &commonpb.KeyValuePair{
Key: common.MmapEnabledKey, Key: common.MmapEnabledKey,
Value: strconv.FormatBool(collectionMmapEnabled), Value: strconv.FormatBool(collectionMmapEnabled),

View File

@ -91,13 +91,14 @@ func ResetFieldProperties(coll *model.Collection, fieldName string, newProps []*
return nil return nil
} }
} }
for _, structField := range coll.StructArrayFields { for i, structField := range coll.StructArrayFields {
if structField.Name == fieldName { if structField.Name == fieldName {
return merr.WrapErrParameterInvalidMsg("struct field has no properties to alter", fieldName) coll.StructArrayFields[i].TypeParams = newProps
return nil
} }
for i, field := range structField.Fields { for j, field := range structField.Fields {
if field.Name == fieldName { if field.Name == fieldName {
structField.Fields[i].TypeParams = newProps coll.StructArrayFields[i].Fields[j].TypeParams = newProps
return nil return nil
} }
} }
@ -113,7 +114,7 @@ func GetFieldProperties(coll *model.Collection, fieldName string) ([]*commonpb.K
} }
for _, structField := range coll.StructArrayFields { for _, structField := range coll.StructArrayFields {
if structField.Name == fieldName { if structField.Name == fieldName {
return nil, merr.WrapErrParameterInvalidMsg("struct field has no properties", fieldName) return structField.TypeParams, nil
} }
for _, field := range structField.Fields { for _, field := range structField.Fields {
if field.Name == fieldName { if field.Name == fieldName {

View File

@ -316,6 +316,28 @@ func FieldHasMmapKey(schema *schemapb.CollectionSchema, fieldID int64) bool {
return false return false
} }
} }
// Check struct array fields
for _, structField := range schema.GetStructArrayFields() {
if structField.GetFieldID() == fieldID {
for _, kv := range structField.GetTypeParams() {
if kv.Key == MmapEnabledKey {
return true
}
}
return false
}
// Check fields inside struct
for _, field := range structField.GetFields() {
if field.GetFieldID() == fieldID {
for _, kv := range field.GetTypeParams() {
if kv.Key == MmapEnabledKey {
return true
}
}
return false
}
}
}
return false return false
} }