mirror of
https://gitee.com/milvus-io/milvus.git
synced 2025-12-07 01:28:27 +08:00
fix: alter collection failed with MMAP setting for STRUCT (#45173)
issue: https://github.com/milvus-io/milvus/issues/45001 ref: https://github.com/milvus-io/milvus/issues/42148 --------- Signed-off-by: SpadeA <tangchenjie1210@gmail.com> Signed-off-by: aoiasd <zhicheng.yue@zilliz.com> Signed-off-by: SpadeA-Tang <tangchenjie1210@gmail.com> Co-authored-by: aoiasd <zhicheng.yue@zilliz.com>
This commit is contained in:
parent
19af1903f6
commit
c0029b788d
@ -1,7 +1,9 @@
|
|||||||
package model
|
package model
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
|
||||||
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
|
"github.com/milvus-io/milvus-proto/go-api/v2/schemapb"
|
||||||
|
"github.com/milvus-io/milvus/pkg/v2/common"
|
||||||
)
|
)
|
||||||
|
|
||||||
type StructArrayField struct {
|
type StructArrayField struct {
|
||||||
@ -9,6 +11,7 @@ type StructArrayField struct {
|
|||||||
Name string
|
Name string
|
||||||
Description string
|
Description string
|
||||||
Fields []*Field
|
Fields []*Field
|
||||||
|
TypeParams []*commonpb.KeyValuePair
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *StructArrayField) Clone() *StructArrayField {
|
func (s *StructArrayField) Clone() *StructArrayField {
|
||||||
@ -17,6 +20,7 @@ func (s *StructArrayField) Clone() *StructArrayField {
|
|||||||
Name: s.Name,
|
Name: s.Name,
|
||||||
Description: s.Description,
|
Description: s.Description,
|
||||||
Fields: CloneFields(s.Fields),
|
Fields: CloneFields(s.Fields),
|
||||||
|
TypeParams: common.CloneKeyValuePairs(s.TypeParams),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -29,10 +33,12 @@ func CloneStructArrayFields(structArrayFields []*StructArrayField) []*StructArra
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (s *StructArrayField) Equal(other StructArrayField) bool {
|
func (s *StructArrayField) Equal(other StructArrayField) bool {
|
||||||
|
var paramsA common.KeyValuePairs = s.TypeParams
|
||||||
return s.FieldID == other.FieldID &&
|
return s.FieldID == other.FieldID &&
|
||||||
s.Name == other.Name &&
|
s.Name == other.Name &&
|
||||||
s.Description == other.Description &&
|
s.Description == other.Description &&
|
||||||
CheckFieldsEqual(s.Fields, other.Fields)
|
CheckFieldsEqual(s.Fields, other.Fields) &&
|
||||||
|
paramsA.Equal(other.TypeParams)
|
||||||
}
|
}
|
||||||
|
|
||||||
func CheckStructArrayFieldsEqual(structArrayFieldsA, structArrayFieldsB []*StructArrayField) bool {
|
func CheckStructArrayFieldsEqual(structArrayFieldsA, structArrayFieldsB []*StructArrayField) bool {
|
||||||
@ -63,6 +69,7 @@ func MarshalStructArrayFieldModel(structArrayField *StructArrayField) *schemapb.
|
|||||||
Name: structArrayField.Name,
|
Name: structArrayField.Name,
|
||||||
Description: structArrayField.Description,
|
Description: structArrayField.Description,
|
||||||
Fields: MarshalFieldModels(structArrayField.Fields),
|
Fields: MarshalFieldModels(structArrayField.Fields),
|
||||||
|
TypeParams: structArrayField.TypeParams,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -88,6 +95,7 @@ func UnmarshalStructArrayFieldModel(fieldSchema *schemapb.StructArrayFieldSchema
|
|||||||
Name: fieldSchema.Name,
|
Name: fieldSchema.Name,
|
||||||
Description: fieldSchema.Description,
|
Description: fieldSchema.Description,
|
||||||
Fields: UnmarshalFieldModels(fieldSchema.Fields),
|
Fields: UnmarshalFieldModels(fieldSchema.Fields),
|
||||||
|
TypeParams: fieldSchema.TypeParams,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@ -285,9 +285,35 @@ func applyCollectionMmapSetting(schema *schemapb.CollectionSchema,
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, structField := range schema.GetStructArrayFields() {
|
for _, structField := range schema.GetStructArrayFields() {
|
||||||
|
structTypeParams := structField.GetTypeParams()
|
||||||
|
structMmapEnabled, structExist := common.IsMmapDataEnabled(structTypeParams...)
|
||||||
|
|
||||||
|
// If struct field itself doesn't have mmap setting, inherit from collection
|
||||||
|
if !structExist && exist &&
|
||||||
|
!common.FieldHasMmapKey(schema, structField.GetFieldID()) {
|
||||||
|
structField.TypeParams = append(structField.TypeParams, &commonpb.KeyValuePair{
|
||||||
|
Key: common.MmapEnabledKey,
|
||||||
|
Value: strconv.FormatBool(collectionMmapEnabled),
|
||||||
|
})
|
||||||
|
// Update struct's mmap state since we just set it
|
||||||
|
structMmapEnabled = collectionMmapEnabled
|
||||||
|
structExist = true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Apply mmap setting to fields inside struct
|
||||||
for _, field := range structField.GetFields() {
|
for _, field := range structField.GetFields() {
|
||||||
if exist &&
|
// Skip if field already has mmap setting
|
||||||
!common.FieldHasMmapKey(schema, field.GetFieldID()) {
|
if common.FieldHasMmapKey(schema, field.GetFieldID()) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
// Priority: struct field setting > collection setting
|
||||||
|
if structExist {
|
||||||
|
field.TypeParams = append(field.TypeParams, &commonpb.KeyValuePair{
|
||||||
|
Key: common.MmapEnabledKey,
|
||||||
|
Value: strconv.FormatBool(structMmapEnabled),
|
||||||
|
})
|
||||||
|
} else if exist {
|
||||||
field.TypeParams = append(field.TypeParams, &commonpb.KeyValuePair{
|
field.TypeParams = append(field.TypeParams, &commonpb.KeyValuePair{
|
||||||
Key: common.MmapEnabledKey,
|
Key: common.MmapEnabledKey,
|
||||||
Value: strconv.FormatBool(collectionMmapEnabled),
|
Value: strconv.FormatBool(collectionMmapEnabled),
|
||||||
|
|||||||
@ -91,13 +91,14 @@ func ResetFieldProperties(coll *model.Collection, fieldName string, newProps []*
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
for _, structField := range coll.StructArrayFields {
|
for i, structField := range coll.StructArrayFields {
|
||||||
if structField.Name == fieldName {
|
if structField.Name == fieldName {
|
||||||
return merr.WrapErrParameterInvalidMsg("struct field has no properties to alter", fieldName)
|
coll.StructArrayFields[i].TypeParams = newProps
|
||||||
|
return nil
|
||||||
}
|
}
|
||||||
for i, field := range structField.Fields {
|
for j, field := range structField.Fields {
|
||||||
if field.Name == fieldName {
|
if field.Name == fieldName {
|
||||||
structField.Fields[i].TypeParams = newProps
|
coll.StructArrayFields[i].Fields[j].TypeParams = newProps
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -113,7 +114,7 @@ func GetFieldProperties(coll *model.Collection, fieldName string) ([]*commonpb.K
|
|||||||
}
|
}
|
||||||
for _, structField := range coll.StructArrayFields {
|
for _, structField := range coll.StructArrayFields {
|
||||||
if structField.Name == fieldName {
|
if structField.Name == fieldName {
|
||||||
return nil, merr.WrapErrParameterInvalidMsg("struct field has no properties", fieldName)
|
return structField.TypeParams, nil
|
||||||
}
|
}
|
||||||
for _, field := range structField.Fields {
|
for _, field := range structField.Fields {
|
||||||
if field.Name == fieldName {
|
if field.Name == fieldName {
|
||||||
|
|||||||
@ -316,6 +316,28 @@ func FieldHasMmapKey(schema *schemapb.CollectionSchema, fieldID int64) bool {
|
|||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
// Check struct array fields
|
||||||
|
for _, structField := range schema.GetStructArrayFields() {
|
||||||
|
if structField.GetFieldID() == fieldID {
|
||||||
|
for _, kv := range structField.GetTypeParams() {
|
||||||
|
if kv.Key == MmapEnabledKey {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
// Check fields inside struct
|
||||||
|
for _, field := range structField.GetFields() {
|
||||||
|
if field.GetFieldID() == fieldID {
|
||||||
|
for _, kv := range field.GetTypeParams() {
|
||||||
|
if kv.Key == MmapEnabledKey {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user