admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-06 17:18:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: Fix bulk import with autoid (#44604)

Browse Source 
issue: #44424

Signed-off-by: sunby <sunbingyi1992@gmail.com>
This commit is contained in:
Bingyi Sun 2025-10-09 12:09:56 +08:00 committed by GitHub
parent 30091a3bb7
commit c25166a202
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 15 additions and 6 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
internal/datanode/importv2/util.go
View File

@ -178,11 +178,7 @@ func AppendSystemFieldsData(task *ImportTask, data *storage.InsertData, rowNum i
}
pkData, ok := data.Data[pkField.GetFieldID()]
allowInsertAutoID, _ := common.IsAllowInsertAutoID(task.req.Schema.GetProperties()...)
if pkField.GetAutoID() {
if allowInsertAutoID && ok && pkData != nil {
// if allowInsertAutoID is true, and pkData is not nil, skip generating primary key data
return nil
}
if pkField.GetAutoID() && (!ok || pkData == nil || pkData.RowNum() == 0 || !allowInsertAutoID) {
switch pkField.GetDataType() {
case schemapb.DataType_Int64:
data.Data[pkField.GetFieldID()] = &storage.Int64FieldData{Data: ids}

9
internal/util/importutilv2/csv/row_parser.go
View File

@ -45,6 +45,7 @@ type rowParser struct {
structArraySubFields map[string]interface{}
pkField *schemapb.FieldSchema
dynamicField *schemapb.FieldSchema
allowInsertAutoID bool
}
func NewRowParser(schema *schemapb.CollectionSchema, header []string, nullkey string) (RowParser, error) {
@ -129,7 +130,6 @@ func NewRowParser(schema *schemapb.CollectionSchema, header []string, nullkey st
fmt.Sprintf("value of field is missed: '%s'", field.GetName()))
}
}
return &rowParser{
nullkey: nullkey,
name2Dim: name2Dim,
@ -139,6 +139,7 @@ func NewRowParser(schema *schemapb.CollectionSchema, header []string, nullkey st
structArraySubFields: structArraySubFields,
pkField: pkField,
dynamicField: dynamicField,
allowInsertAutoID: allowInsertAutoID,
}, nil
}
@ -235,6 +236,12 @@ func (r *rowParser) Parse(strArr []string) (Row, error) {
row[field.GetFieldID()] = data
} else if r.dynamicField != nil {
dynamicValues[r.header[index]] = value
} else if r.pkField.GetName() == r.header[index] && r.pkField.GetAutoID() && r.allowInsertAutoID {
data, err := r.parseEntity(r.pkField, value, false)
if err != nil {
return nil, err
}
row[r.pkField.GetFieldID()] = data
} else {
// from v2.6, we don't intend to return error for redundant fields, just skip it
continue

6
internal/util/importutilv2/json/row_parser.go
View File

@ -182,6 +182,12 @@ func (r *rowParser) Parse(raw any) (Row, error) {
return err
}
row[fieldID] = data
} else if r.pkField.GetName() == key && r.pkField.GetAutoID() && r.allowInsertAutoID {
data, err := r.parseEntity(r.pkField.GetFieldID(), value)
if err != nil {
return err
}
row[r.pkField.GetFieldID()] = data
} else if r.dynamicField != nil {
// has dynamic field, put redundant pair to dynamicValues
dynamicValues[key] = value