mirror of
https://gitee.com/milvus-io/milvus.git
synced 2025-12-29 06:55:27 +08:00
f6f716bcfd
issue: https://github.com/milvus-io/milvus/issues/42148 For a vector field inside a STRUCT, since a STRUCT can only appear as the element type of an ARRAY field, the vector field in STRUCT is effectively an array of vectors, i.e. an embedding list. Milvus already supports searching embedding lists with metrics whose names start with the prefix MAX_SIM_. This PR allows Milvus to search embeddings inside an embedding list using the same metrics as normal embedding fields. Each embedding in the list is treated as an independent vector and participates in ANN search. Further, since STRUCT may contain scalar fields that are highly related to the embedding field, this PR introduces an element-level filter expression to refine search results. The grammar of the element-level filter is: element_filter(structFieldName, $[subFieldName] == 3) where $[subFieldName] refers to the value of subFieldName in each element of the STRUCT array structFieldName. It can be combined with existing filter expressions, for example: "varcharField == 'aaa' && element_filter(struct_field, $[struct_int] == 3)" A full example: ``` struct_schema = milvus_client.create_struct_field_schema() struct_schema.add_field("struct_str", DataType.VARCHAR, max_length=65535) struct_schema.add_field("struct_int", DataType.INT32) struct_schema.add_field("struct_float_vec", DataType.FLOAT_VECTOR, dim=EMBEDDING_DIM) schema.add_field( "struct_field", datatype=DataType.ARRAY, element_type=DataType.STRUCT, struct_schema=struct_schema, max_capacity=1000, ) ... filter = "varcharField == 'aaa' && element_filter(struct_field, $[struct_int] == 3 && $[struct_str] == 'abc')" res = milvus_client.search( COLLECTION_NAME, data=query_embeddings, limit=10, anns_field="struct_field[struct_float_vec]", filter=filter, output_fields=["struct_field[struct_int]", "varcharField"], ) ``` TODO: 1. When an `element_filter` expression is used, a regular filter expression must also be present. Remove this restriction. 2. Implement `element_filter` expressions in the `query`. --------- Signed-off-by: SpadeA <tangchenjie1210@gmail.com>
199 lines
9.2 KiB
ANTLR
199 lines
9.2 KiB
ANTLR
grammar Plan;
|
|
|
|
expr:
|
|
Identifier (op1=(ADD | SUB) INTERVAL interval_string=StringLiteral)? op2=(LT | LE | GT | GE | EQ | NE) ISO compare_string=StringLiteral # TimestamptzCompareForward
|
|
| ISO compare_string=StringLiteral op2=(LT | LE | GT | GE | EQ | NE) Identifier (op1=(ADD | SUB) INTERVAL interval_string=StringLiteral)? # TimestamptzCompareReverse
|
|
| IntegerConstant # Integer
|
|
| FloatingConstant # Floating
|
|
| BooleanConstant # Boolean
|
|
| StringLiteral # String
|
|
| (Identifier|Meta) # Identifier
|
|
| JSONIdentifier # JSONIdentifier
|
|
| StructSubFieldIdentifier # StructSubField
|
|
| LBRACE Identifier RBRACE # TemplateVariable
|
|
| '(' expr ')' # Parens
|
|
| '[' expr (',' expr)* ','? ']' # Array
|
|
| EmptyArray # EmptyArray
|
|
| EXISTS expr # Exists
|
|
| expr LIKE StringLiteral # Like
|
|
| TEXTMATCH'('Identifier',' StringLiteral (',' textMatchOption)? ')' # TextMatch
|
|
| PHRASEMATCH'('Identifier',' StringLiteral (',' expr)? ')' # PhraseMatch
|
|
| RANDOMSAMPLE'(' expr ')' # RandomSample
|
|
| ElementFilter'('Identifier',' expr')' # ElementFilter
|
|
| expr POW expr # Power
|
|
| op = (ADD | SUB | BNOT | NOT) expr # Unary
|
|
// | '(' typeName ')' expr # Cast
|
|
| expr op = (MUL | DIV | MOD) expr # MulDivMod
|
|
| expr op = (ADD | SUB) expr # AddSub
|
|
| expr op = (SHL | SHR) expr # Shift
|
|
| expr op = NOT? IN expr # Term
|
|
| (JSONContains | ArrayContains)'('expr',' expr')' # JSONContains
|
|
| (JSONContainsAll | ArrayContainsAll)'('expr',' expr')' # JSONContainsAll
|
|
| (JSONContainsAny | ArrayContainsAny)'('expr',' expr')' # JSONContainsAny
|
|
| STEuqals'('Identifier','StringLiteral')' # STEuqals
|
|
| STTouches'('Identifier','StringLiteral')' # STTouches
|
|
| STOverlaps'('Identifier','StringLiteral')' # STOverlaps
|
|
| STCrosses'('Identifier','StringLiteral')' # STCrosses
|
|
| STContains'('Identifier','StringLiteral')' # STContains
|
|
| STIntersects'('Identifier','StringLiteral')' # STIntersects
|
|
| STWithin'('Identifier','StringLiteral')' # STWithin
|
|
| STDWithin'('Identifier','StringLiteral',' expr')' # STDWithin
|
|
| STIsValid'('Identifier')' # STIsValid
|
|
| ArrayLength'('(Identifier | JSONIdentifier)')' # ArrayLength
|
|
| Identifier '(' ( expr (',' expr )* ','? )? ')' # Call
|
|
| expr op1 = (LT | LE) (Identifier | JSONIdentifier | StructSubFieldIdentifier) op2 = (LT | LE) expr # Range
|
|
| expr op1 = (GT | GE) (Identifier | JSONIdentifier | StructSubFieldIdentifier) op2 = (GT | GE) expr # ReverseRange
|
|
| expr op = (LT | LE | GT | GE) expr # Relational
|
|
| expr op = (EQ | NE) expr # Equality
|
|
| expr BAND expr # BitAnd
|
|
| expr BXOR expr # BitXor
|
|
| expr BOR expr # BitOr
|
|
| expr AND expr # LogicalAnd
|
|
| expr OR expr # LogicalOr
|
|
| (Identifier | JSONIdentifier) ISNULL # IsNull
|
|
| (Identifier | JSONIdentifier) ISNOTNULL # IsNotNull;
|
|
|
|
textMatchOption:
|
|
MINIMUM_SHOULD_MATCH ASSIGN IntegerConstant;
|
|
|
|
// typeName: ty = (BOOL | INT8 | INT16 | INT32 | INT64 | FLOAT | DOUBLE);
|
|
|
|
// BOOL: 'bool';
|
|
// INT8: 'int8';
|
|
// INT16: 'int16';
|
|
// INT32: 'int32';
|
|
// INT64: 'int64';
|
|
// FLOAT: 'float';
|
|
// DOUBLE: 'double';
|
|
LBRACE: '{';
|
|
RBRACE: '}';
|
|
|
|
LT: '<';
|
|
LE: '<=';
|
|
GT: '>';
|
|
GE: '>=';
|
|
EQ: '==';
|
|
NE: '!=';
|
|
|
|
LIKE: 'like' | 'LIKE';
|
|
EXISTS: 'exists' | 'EXISTS';
|
|
TEXTMATCH: 'text_match'|'TEXT_MATCH';
|
|
PHRASEMATCH: 'phrase_match'|'PHRASE_MATCH';
|
|
RANDOMSAMPLE: 'random_sample' | 'RANDOM_SAMPLE';
|
|
INTERVAL: 'interval' | 'INTERVAL';
|
|
ISO: 'iso' | 'ISO';
|
|
MINIMUM_SHOULD_MATCH: 'minimum_should_match' | 'MINIMUM_SHOULD_MATCH';
|
|
ASSIGN: '=';
|
|
|
|
ADD: '+';
|
|
SUB: '-';
|
|
MUL: '*';
|
|
DIV: '/';
|
|
MOD: '%';
|
|
POW: '**';
|
|
SHL: '<<';
|
|
SHR: '>>';
|
|
BAND: '&';
|
|
BOR: '|';
|
|
BXOR: '^';
|
|
|
|
AND: '&&' | 'and' | 'AND';
|
|
OR: '||' | 'or' | 'OR';
|
|
|
|
ISNULL: 'is null' | 'IS NULL';
|
|
ISNOTNULL: 'is not null' | 'IS NOT NULL';
|
|
|
|
BNOT: '~';
|
|
NOT: '!' | 'not' | 'NOT';
|
|
|
|
IN: 'in' | 'IN';
|
|
EmptyArray: '[' (Whitespace | Newline)* ']';
|
|
|
|
JSONContains: 'json_contains' | 'JSON_CONTAINS';
|
|
JSONContainsAll: 'json_contains_all' | 'JSON_CONTAINS_ALL';
|
|
JSONContainsAny: 'json_contains_any' | 'JSON_CONTAINS_ANY';
|
|
|
|
ArrayContains: 'array_contains' | 'ARRAY_CONTAINS';
|
|
ArrayContainsAll: 'array_contains_all' | 'ARRAY_CONTAINS_ALL';
|
|
ArrayContainsAny: 'array_contains_any' | 'ARRAY_CONTAINS_ANY';
|
|
ArrayLength: 'array_length' | 'ARRAY_LENGTH';
|
|
ElementFilter: 'element_filter' | 'ELEMENT_FILTER';
|
|
|
|
STEuqals:'st_equals' | 'ST_EQUALS';
|
|
STTouches:'st_touches' | 'ST_TOUCHES';
|
|
STOverlaps: 'st_overlaps' | 'ST_OVERLAPS';
|
|
STCrosses: 'st_crosses' | 'ST_CROSSES';
|
|
STContains: 'st_contains' | 'ST_CONTAINS';
|
|
STIntersects : 'st_intersects' | 'ST_INTERSECTS';
|
|
STWithin :'st_within' | 'ST_WITHIN';
|
|
STDWithin: 'st_dwithin' | 'ST_DWITHIN';
|
|
STIsValid: 'st_isvalid' | 'ST_ISVALID';
|
|
|
|
BooleanConstant: 'true' | 'True' | 'TRUE' | 'false' | 'False' | 'FALSE';
|
|
|
|
IntegerConstant:
|
|
DecimalConstant
|
|
| OctalConstant
|
|
| HexadecimalConstant
|
|
| BinaryConstant;
|
|
|
|
FloatingConstant:
|
|
DecimalFloatingConstant
|
|
| HexadecimalFloatingConstant;
|
|
|
|
Identifier: Nondigit (Nondigit | Digit)*;
|
|
Meta: '$meta';
|
|
|
|
StringLiteral: EncodingPrefix? ('"' DoubleSCharSequence? '"' | '\'' SingleSCharSequence? '\'');
|
|
JSONIdentifier: (Identifier | Meta)('[' (StringLiteral | DecimalConstant) ']')+;
|
|
StructSubFieldIdentifier: '$[' Identifier ']';
|
|
|
|
fragment EncodingPrefix: 'u8' | 'u' | 'U' | 'L';
|
|
|
|
fragment DoubleSCharSequence: DoubleSChar+;
|
|
fragment SingleSCharSequence: SingleSChar+;
|
|
|
|
fragment DoubleSChar: ~["\\\r\n] | EscapeSequence | '\\\n' | '\\\r\n';
|
|
fragment SingleSChar: ~['\\\r\n] | EscapeSequence | '\\\n' | '\\\r\n';
|
|
fragment Nondigit: [a-zA-Z_];
|
|
fragment Digit: [0-9];
|
|
fragment BinaryConstant: '0' [bB] [0-1]+;
|
|
fragment DecimalConstant: NonzeroDigit Digit* | '0';
|
|
fragment OctalConstant: '0' OctalDigit*;
|
|
fragment HexadecimalConstant: '0' [xX] HexadecimalDigitSequence;
|
|
fragment NonzeroDigit: [1-9];
|
|
fragment OctalDigit: [0-7];
|
|
fragment HexadecimalDigit: [0-9a-fA-F];
|
|
fragment HexQuad:
|
|
HexadecimalDigit HexadecimalDigit HexadecimalDigit HexadecimalDigit;
|
|
fragment UniversalCharacterName:
|
|
'\\u' HexQuad
|
|
| '\\U' HexQuad HexQuad;
|
|
fragment DecimalFloatingConstant:
|
|
FractionalConstant ExponentPart?
|
|
| DigitSequence ExponentPart;
|
|
fragment HexadecimalFloatingConstant:
|
|
'0' [xX] (
|
|
HexadecimalFractionalConstant
|
|
| HexadecimalDigitSequence
|
|
) BinaryExponentPart;
|
|
fragment FractionalConstant:
|
|
DigitSequence? '.' DigitSequence
|
|
| DigitSequence '.';
|
|
fragment ExponentPart: [eE] [+-]? DigitSequence;
|
|
fragment DigitSequence: Digit+;
|
|
fragment HexadecimalFractionalConstant:
|
|
HexadecimalDigitSequence? '.' HexadecimalDigitSequence
|
|
| HexadecimalDigitSequence '.';
|
|
fragment HexadecimalDigitSequence: HexadecimalDigit+;
|
|
fragment BinaryExponentPart: [pP] [+-]? DigitSequence;
|
|
fragment EscapeSequence:
|
|
'\\' ['"?abfnrtv\\]
|
|
| '\\' OctalDigit OctalDigit? OctalDigit?
|
|
| '\\x' HexadecimalDigitSequence
|
|
| UniversalCharacterName;
|
|
|
|
Whitespace: [ \t]+ -> skip;
|
|
|
|
Newline: ( '\r' '\n'? | '\n') -> skip;
|