mirror of
https://gitee.com/milvus-io/milvus.git
synced 2026-01-07 19:31:51 +08:00
f6f716bcfd
issue: https://github.com/milvus-io/milvus/issues/42148 For a vector field inside a STRUCT, since a STRUCT can only appear as the element type of an ARRAY field, the vector field in STRUCT is effectively an array of vectors, i.e. an embedding list. Milvus already supports searching embedding lists with metrics whose names start with the prefix MAX_SIM_. This PR allows Milvus to search embeddings inside an embedding list using the same metrics as normal embedding fields. Each embedding in the list is treated as an independent vector and participates in ANN search. Further, since STRUCT may contain scalar fields that are highly related to the embedding field, this PR introduces an element-level filter expression to refine search results. The grammar of the element-level filter is: element_filter(structFieldName, $[subFieldName] == 3) where $[subFieldName] refers to the value of subFieldName in each element of the STRUCT array structFieldName. It can be combined with existing filter expressions, for example: "varcharField == 'aaa' && element_filter(struct_field, $[struct_int] == 3)" A full example: ``` struct_schema = milvus_client.create_struct_field_schema() struct_schema.add_field("struct_str", DataType.VARCHAR, max_length=65535) struct_schema.add_field("struct_int", DataType.INT32) struct_schema.add_field("struct_float_vec", DataType.FLOAT_VECTOR, dim=EMBEDDING_DIM) schema.add_field( "struct_field", datatype=DataType.ARRAY, element_type=DataType.STRUCT, struct_schema=struct_schema, max_capacity=1000, ) ... filter = "varcharField == 'aaa' && element_filter(struct_field, $[struct_int] == 3 && $[struct_str] == 'abc')" res = milvus_client.search( COLLECTION_NAME, data=query_embeddings, limit=10, anns_field="struct_field[struct_float_vec]", filter=filter, output_fields=["struct_field[struct_int]", "varcharField"], ) ``` TODO: 1. When an `element_filter` expression is used, a regular filter expression must also be present. Remove this restriction. 2. Implement `element_filter` expressions in the `query`. --------- Signed-off-by: SpadeA <tangchenjie1210@gmail.com>
559 lines
13 KiB
C++
559 lines
13 KiB
C++
// Licensed to the LF AI & Data foundation under one
|
|
// or more contributor license agreements. See the NOTICE file
|
|
// distributed with this work for additional information
|
|
// regarding copyright ownership. The ASF licenses this file
|
|
// to you under the Apache License, Version 2.0 (the
|
|
// "License"); you may not use this file except in compliance
|
|
// with the License. You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
#pragma once
|
|
|
|
#include <memory>
|
|
#include <string>
|
|
#include <vector>
|
|
|
|
#include "common/Types.h"
|
|
#include "common/Vector.h"
|
|
#include "expr/ITypeExpr.h"
|
|
#include "common/EasyAssert.h"
|
|
#include "pb/plan.pb.h"
|
|
#include "segcore/SegmentInterface.h"
|
|
#include "plan/PlanNodeIdGenerator.h"
|
|
#include "rescores/Scorer.h"
|
|
|
|
namespace milvus {
|
|
namespace plan {
|
|
|
|
typedef std::string PlanNodeId;
|
|
/**
|
|
* @brief Base class for all logic plan node
|
|
*
|
|
*/
|
|
class PlanNode {
|
|
public:
|
|
explicit PlanNode(const PlanNodeId& id) : id_(id) {
|
|
}
|
|
|
|
virtual ~PlanNode() = default;
|
|
|
|
const PlanNodeId&
|
|
id() const {
|
|
return id_;
|
|
}
|
|
|
|
virtual DataType
|
|
output_type() const = 0;
|
|
|
|
virtual std::vector<std::shared_ptr<PlanNode>>
|
|
sources() const = 0;
|
|
|
|
virtual bool
|
|
RequireSplits() const {
|
|
return false;
|
|
}
|
|
|
|
virtual std::string
|
|
ToString() const = 0;
|
|
|
|
virtual std::string_view
|
|
name() const = 0;
|
|
|
|
virtual expr::ExprInfo
|
|
GatherInfo() const {
|
|
return {};
|
|
};
|
|
|
|
std::string
|
|
SourceToString() const {
|
|
std::vector<std::string> sources_str;
|
|
for (auto& source : sources()) {
|
|
sources_str.emplace_back(source->ToString());
|
|
}
|
|
return "[" + Join(sources_str, ",") + "]";
|
|
}
|
|
|
|
private:
|
|
PlanNodeId id_;
|
|
};
|
|
|
|
using PlanNodePtr = std::shared_ptr<PlanNode>;
|
|
|
|
class FilterNode : public PlanNode {
|
|
public:
|
|
FilterNode(const PlanNodeId& id,
|
|
expr::TypedExprPtr filter,
|
|
std::vector<PlanNodePtr> sources)
|
|
: PlanNode(id),
|
|
sources_{std::move(sources)},
|
|
filter_(std::move(filter)) {
|
|
AssertInfo(
|
|
filter_->type() == DataType::BOOL,
|
|
fmt::format("Filter expression must be of type BOOLEAN, Got {}",
|
|
filter_->type()));
|
|
}
|
|
|
|
DataType
|
|
output_type() const override {
|
|
return sources_[0]->output_type();
|
|
}
|
|
|
|
std::vector<PlanNodePtr>
|
|
sources() const override {
|
|
return sources_;
|
|
}
|
|
|
|
const expr::TypedExprPtr&
|
|
filter() const {
|
|
return filter_;
|
|
}
|
|
|
|
std::string_view
|
|
name() const override {
|
|
return "Filter";
|
|
}
|
|
|
|
std::string
|
|
ToString() const override {
|
|
return "";
|
|
}
|
|
|
|
private:
|
|
const std::vector<PlanNodePtr> sources_;
|
|
const expr::TypedExprPtr filter_;
|
|
};
|
|
|
|
class FilterBitsNode : public PlanNode {
|
|
public:
|
|
FilterBitsNode(
|
|
const PlanNodeId& id,
|
|
expr::TypedExprPtr filter,
|
|
std::vector<PlanNodePtr> sources = std::vector<PlanNodePtr>{})
|
|
: PlanNode(id),
|
|
sources_{std::move(sources)},
|
|
filter_(std::move(filter)) {
|
|
AssertInfo(
|
|
filter_->type() == DataType::BOOL,
|
|
fmt::format("Filter expression must be of type BOOLEAN, Got {}",
|
|
filter_->type()));
|
|
}
|
|
|
|
DataType
|
|
output_type() const override {
|
|
return DataType::BOOL;
|
|
}
|
|
|
|
std::vector<PlanNodePtr>
|
|
sources() const override {
|
|
return sources_;
|
|
}
|
|
|
|
const expr::TypedExprPtr&
|
|
filter() const {
|
|
return filter_;
|
|
}
|
|
|
|
std::string_view
|
|
name() const override {
|
|
return "FilterBits";
|
|
}
|
|
|
|
std::string
|
|
ToString() const override {
|
|
return fmt::format("FilterBitsNode:\n\t[filter_expr:{}]",
|
|
filter_->ToString());
|
|
}
|
|
|
|
expr::ExprInfo
|
|
GatherInfo() const override {
|
|
expr::ExprInfo info;
|
|
filter_->GatherInfo(info);
|
|
return info;
|
|
}
|
|
|
|
private:
|
|
const std::vector<PlanNodePtr> sources_;
|
|
const expr::TypedExprPtr filter_;
|
|
};
|
|
|
|
class ElementFilterNode : public PlanNode {
|
|
public:
|
|
ElementFilterNode(const PlanNodeId& id,
|
|
expr::TypedExprPtr element_filter,
|
|
std::string struct_name,
|
|
std::vector<PlanNodePtr> sources)
|
|
: PlanNode(id),
|
|
sources_{std::move(sources)},
|
|
element_filter_(std::move(element_filter)),
|
|
struct_name_(std::move(struct_name)) {
|
|
AssertInfo(
|
|
element_filter_->type() == DataType::BOOL,
|
|
fmt::format(
|
|
"Element filter expression must be of type BOOLEAN, Got {}",
|
|
element_filter_->type()));
|
|
}
|
|
|
|
DataType
|
|
output_type() const override {
|
|
return DataType::NONE;
|
|
}
|
|
|
|
std::vector<PlanNodePtr>
|
|
sources() const override {
|
|
return sources_;
|
|
}
|
|
|
|
const expr::TypedExprPtr&
|
|
element_filter() const {
|
|
return element_filter_;
|
|
}
|
|
|
|
const std::string&
|
|
struct_name() const {
|
|
return struct_name_;
|
|
}
|
|
|
|
std::string_view
|
|
name() const override {
|
|
return "ElementFilter";
|
|
}
|
|
|
|
std::string
|
|
ToString() const override {
|
|
return fmt::format(
|
|
"ElementFilterNode:\n\t[struct_name:{}, element_filter:{}]",
|
|
struct_name_,
|
|
element_filter_->ToString());
|
|
}
|
|
|
|
private:
|
|
const std::vector<PlanNodePtr> sources_;
|
|
const expr::TypedExprPtr element_filter_;
|
|
const std::string struct_name_;
|
|
};
|
|
|
|
class ElementFilterBitsNode : public PlanNode {
|
|
public:
|
|
ElementFilterBitsNode(
|
|
const PlanNodeId& id,
|
|
expr::TypedExprPtr element_filter,
|
|
std::string struct_name,
|
|
std::vector<PlanNodePtr> sources = std::vector<PlanNodePtr>{})
|
|
: PlanNode(id),
|
|
sources_{std::move(sources)},
|
|
element_filter_(std::move(element_filter)),
|
|
struct_name_(std::move(struct_name)) {
|
|
AssertInfo(
|
|
element_filter_->type() == DataType::BOOL,
|
|
fmt::format(
|
|
"Element filter expression must be of type BOOLEAN, Got {}",
|
|
element_filter_->type()));
|
|
}
|
|
|
|
DataType
|
|
output_type() const override {
|
|
return DataType::BOOL;
|
|
}
|
|
|
|
std::vector<PlanNodePtr>
|
|
sources() const override {
|
|
return sources_;
|
|
}
|
|
|
|
const expr::TypedExprPtr&
|
|
element_filter() const {
|
|
return element_filter_;
|
|
}
|
|
|
|
const std::string&
|
|
struct_name() const {
|
|
return struct_name_;
|
|
}
|
|
|
|
std::string_view
|
|
name() const override {
|
|
return "ElementFilterBits";
|
|
}
|
|
|
|
std::string
|
|
ToString() const override {
|
|
return fmt::format(
|
|
"ElementFilterBitsNode:\n\t[struct_name:{}, element_filter:{}]",
|
|
struct_name_,
|
|
element_filter_->ToString());
|
|
}
|
|
|
|
expr::ExprInfo
|
|
GatherInfo() const override {
|
|
expr::ExprInfo info;
|
|
element_filter_->GatherInfo(info);
|
|
return info;
|
|
}
|
|
|
|
private:
|
|
const std::vector<PlanNodePtr> sources_;
|
|
const expr::TypedExprPtr element_filter_;
|
|
const std::string struct_name_;
|
|
};
|
|
|
|
class MvccNode : public PlanNode {
|
|
public:
|
|
MvccNode(const PlanNodeId& id,
|
|
std::vector<PlanNodePtr> sources = std::vector<PlanNodePtr>{})
|
|
: PlanNode(id), sources_{std::move(sources)} {
|
|
}
|
|
|
|
DataType
|
|
output_type() const override {
|
|
return DataType::BOOL;
|
|
}
|
|
|
|
std::vector<PlanNodePtr>
|
|
sources() const override {
|
|
return sources_;
|
|
}
|
|
|
|
std::string_view
|
|
name() const override {
|
|
return "MvccNode";
|
|
}
|
|
|
|
std::string
|
|
ToString() const override {
|
|
return fmt::format("MvccNode:\n\t[source node:{}]", SourceToString());
|
|
}
|
|
|
|
private:
|
|
const std::vector<PlanNodePtr> sources_;
|
|
};
|
|
|
|
class RandomSampleNode : public PlanNode {
|
|
public:
|
|
RandomSampleNode(
|
|
const PlanNodeId& id,
|
|
float factor,
|
|
std::vector<PlanNodePtr> sources = std::vector<PlanNodePtr>{})
|
|
: PlanNode(id), factor_(factor), sources_(std::move(sources)) {
|
|
}
|
|
|
|
DataType
|
|
output_type() const override {
|
|
return DataType::BOOL;
|
|
}
|
|
|
|
std::vector<PlanNodePtr>
|
|
sources() const override {
|
|
return sources_;
|
|
}
|
|
|
|
std::string_view
|
|
name() const override {
|
|
return "RandomSampleNode";
|
|
}
|
|
|
|
std::string
|
|
ToString() const override {
|
|
return fmt::format("RandomSampleNode:\n\t[factor:{}]", factor_);
|
|
}
|
|
|
|
float
|
|
factor() const {
|
|
return factor_;
|
|
}
|
|
|
|
private:
|
|
float factor_;
|
|
const std::vector<PlanNodePtr> sources_;
|
|
};
|
|
|
|
class VectorSearchNode : public PlanNode {
|
|
public:
|
|
VectorSearchNode(
|
|
const PlanNodeId& id,
|
|
std::vector<PlanNodePtr> sources = std::vector<PlanNodePtr>{})
|
|
: PlanNode(id), sources_{std::move(sources)} {
|
|
}
|
|
|
|
DataType
|
|
output_type() const override {
|
|
return DataType::BOOL;
|
|
}
|
|
|
|
std::vector<PlanNodePtr>
|
|
sources() const override {
|
|
return sources_;
|
|
}
|
|
|
|
std::string_view
|
|
name() const override {
|
|
return "VectorSearchNode";
|
|
}
|
|
|
|
std::string
|
|
ToString() const override {
|
|
return fmt::format("VectorSearchNode:\n\t[source node:{}]",
|
|
SourceToString());
|
|
}
|
|
|
|
private:
|
|
const std::vector<PlanNodePtr> sources_;
|
|
};
|
|
|
|
class GroupByNode : public PlanNode {
|
|
public:
|
|
GroupByNode(const PlanNodeId& id,
|
|
std::vector<PlanNodePtr> sources = std::vector<PlanNodePtr>{})
|
|
: PlanNode(id), sources_{std::move(sources)} {
|
|
}
|
|
|
|
DataType
|
|
output_type() const override {
|
|
return DataType::BOOL;
|
|
}
|
|
|
|
std::vector<PlanNodePtr>
|
|
sources() const override {
|
|
return sources_;
|
|
}
|
|
|
|
std::string_view
|
|
name() const override {
|
|
return "GroupByNode";
|
|
}
|
|
|
|
std::string
|
|
ToString() const override {
|
|
return fmt::format("GroupByNode:\n\t[source node:{}]",
|
|
SourceToString());
|
|
}
|
|
|
|
private:
|
|
const std::vector<PlanNodePtr> sources_;
|
|
};
|
|
|
|
class CountNode : public PlanNode {
|
|
public:
|
|
CountNode(
|
|
const PlanNodeId& id,
|
|
const std::vector<PlanNodePtr>& sources = std::vector<PlanNodePtr>{})
|
|
: PlanNode(id), sources_{std::move(sources)} {
|
|
}
|
|
|
|
DataType
|
|
output_type() const override {
|
|
return DataType::INT64;
|
|
}
|
|
|
|
std::vector<PlanNodePtr>
|
|
sources() const override {
|
|
return sources_;
|
|
}
|
|
|
|
std::string_view
|
|
name() const override {
|
|
return "CountNode";
|
|
}
|
|
|
|
std::string
|
|
ToString() const override {
|
|
return fmt::format("CountNode:\n\t[source node:{}]", SourceToString());
|
|
}
|
|
|
|
private:
|
|
const std::vector<PlanNodePtr> sources_;
|
|
};
|
|
|
|
class RescoresNode : public PlanNode {
|
|
public:
|
|
RescoresNode(
|
|
const PlanNodeId& id,
|
|
const std::vector<std::shared_ptr<rescores::Scorer>>& scorers,
|
|
const proto::plan::ScoreOption& option,
|
|
const std::vector<PlanNodePtr>& sources = std::vector<PlanNodePtr>{})
|
|
: PlanNode(id),
|
|
scorers_(std::move(scorers)),
|
|
option_(std::move(option)),
|
|
sources_{std::move(sources)} {
|
|
}
|
|
|
|
DataType
|
|
output_type() const override {
|
|
return DataType::INT64;
|
|
}
|
|
|
|
std::vector<PlanNodePtr>
|
|
sources() const override {
|
|
return sources_;
|
|
}
|
|
|
|
const proto::plan::ScoreOption*
|
|
option() const {
|
|
return &option_;
|
|
}
|
|
|
|
const std::vector<std::shared_ptr<rescores::Scorer>>&
|
|
scorers() const {
|
|
return scorers_;
|
|
}
|
|
|
|
std::string_view
|
|
name() const override {
|
|
return "RescoresNode";
|
|
}
|
|
|
|
std::string
|
|
ToString() const override {
|
|
return fmt::format("RescoresNode:\n\t[source node:{}]",
|
|
SourceToString());
|
|
}
|
|
|
|
private:
|
|
const proto::plan::ScoreOption option_;
|
|
const std::vector<PlanNodePtr> sources_;
|
|
const std::vector<std::shared_ptr<rescores::Scorer>> scorers_;
|
|
};
|
|
|
|
enum class ExecutionStrategy {
|
|
// Process splits as they come in any available driver.
|
|
kUngrouped,
|
|
// Process splits from each split group only in one driver.
|
|
// It is used when split groups represent separate partitions of the data on
|
|
// the grouping keys or join keys. In that case it is sufficient to keep only
|
|
// the keys from a single split group in a hash table used by group-by or
|
|
// join.
|
|
kGrouped,
|
|
};
|
|
struct PlanFragment {
|
|
std::shared_ptr<const PlanNode> plan_node_;
|
|
ExecutionStrategy execution_strategy_{ExecutionStrategy::kUngrouped};
|
|
int32_t num_splitgroups_{0};
|
|
|
|
PlanFragment() = default;
|
|
|
|
inline bool
|
|
IsGroupedExecution() const {
|
|
return execution_strategy_ == ExecutionStrategy::kGrouped;
|
|
}
|
|
|
|
explicit PlanFragment(std::shared_ptr<const PlanNode> top_node,
|
|
ExecutionStrategy strategy,
|
|
int32_t num_splitgroups)
|
|
: plan_node_(std::move(top_node)),
|
|
execution_strategy_(strategy),
|
|
num_splitgroups_(num_splitgroups) {
|
|
}
|
|
|
|
explicit PlanFragment(std::shared_ptr<const PlanNode> top_node)
|
|
: plan_node_(std::move(top_node)) {
|
|
}
|
|
};
|
|
|
|
} // namespace plan
|
|
} // namespace milvus
|