admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-28 22:45:26 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: char_group tokenizer only support one byte char as delimiters (#46193)

Browse Source 
relate: https://github.com/milvus-io/milvus/issues/46192

Signed-off-by: aoiasd <zhicheng.yue@zilliz.com>
This commit is contained in:
aoiasd 2025-12-10 14:33:13 +08:00 committed by GitHub
parent 6e2872c982
commit c84b6d56f8
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 2 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
internal/core/thirdparty/tantivy/tantivy-binding/src/analyzer/tokenizers/char_group_tokenizer.rs vendored
View File

@ -85,7 +85,7 @@ impl CharGroupTokenizer {
.to_string(),
)),
|v| {
if v.len() == 1 {
if v.chars().count() == 1 {
delimiters.insert(v.chars().next().unwrap());
return Ok(());
}
@ -204,7 +204,7 @@ mod tests {
fn test_char_group_tokenizer() {
let params = r#"{
"type": "chargroup",
"delimiters": ["o", "punctuation","digit"]
"delimiters": ["o", "punctuation","digit", "，"]
}"#;
let json_param = json::from_str::<json::Map<String, json::Value>>(&params);
assert!(json_param.is_ok());