From aaced54f5d702910349e2e5a5fc5343ba62b5edd Mon Sep 17 00:00:00 2001
From: jaime
Date: Thu, 27 Feb 2025 09:30:42 +0800
Subject: [PATCH] =?UTF-8?q?feat:=20=E4=BC=98=E5=8C=96=E7=B4=A2=E5=BC=95?=
=?UTF-8?q?=E5=88=9B=E5=BB=BA?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
---
.../dromara/easyes/common/join/BaseJoin.java | 8 +-
.../common/utils/jackson/JsonUtils.java | 1 +
.../easyes/core/conditions/function/Func.java | 12 +-
.../core/kernel/AbstractChainWrapper.java | 4 +-
.../easyes/core/kernel/AbstractWrapper.java | 4 +-
.../easyes/core/kernel/BaseEsMapper.java | 10 +
.../easyes/core/kernel/BaseEsMapperImpl.java | 34 +--
.../easyes/core/kernel/WrapperProcessor.java | 83 +++---
.../dromara/easyes/core/toolkit/GeoUtils.java | 208 ++++++++++++++-
.../easyes/core/toolkit/IndexUtils.java | 8 +
.../dromara/easyes/test/entity/Document.java | 2 +-
.../org/dromara/easyes/test/all/AllTest.java | 237 +++++++++---------
.../easyes/test/all/XmlScannerAllTest.java | 6 +-
.../org/dromara/easyes/test/all/AllTest.java | 6 +-
.../org/dromara/easyes/test/mix/MixTest.java | 10 +-
.../easyes/test/vector/VectorTest.java | 2 +-
16 files changed, 423 insertions(+), 212 deletions(-)
diff --git a/easy-es-common/src/main/java/org/dromara/easyes/common/join/BaseJoin.java b/easy-es-common/src/main/java/org/dromara/easyes/common/join/BaseJoin.java
index be5f69cc..fbf5a5e8 100644
--- a/easy-es-common/src/main/java/org/dromara/easyes/common/join/BaseJoin.java
+++ b/easy-es-common/src/main/java/org/dromara/easyes/common/join/BaseJoin.java
@@ -1,6 +1,7 @@
package org.dromara.easyes.common.join;
import com.fasterxml.jackson.annotation.JsonAnyGetter;
+import com.fasterxml.jackson.annotation.JsonIgnore;
import lombok.Setter;
import java.util.HashMap;
@@ -9,11 +10,12 @@ import java.util.Map;
@Setter
public class BaseJoin {
- public Map> joinFiled;
+ @JsonIgnore
+ public Map> joinField;
@JsonAnyGetter
public Map> getJoinField() {
- return joinFiled;
+ return joinField;
}
/**
@@ -28,7 +30,7 @@ public class BaseJoin {
* @param parentId 父id
*/
public void addJoinField(String fieldName, String name, String parentId) {
- joinFiled = new HashMap<>() {{
+ joinField = new HashMap<>() {{
put(fieldName, new HashMap<>() {{
put("name", name);
if (parentId != null) {
diff --git a/easy-es-common/src/main/java/org/dromara/easyes/common/utils/jackson/JsonUtils.java b/easy-es-common/src/main/java/org/dromara/easyes/common/utils/jackson/JsonUtils.java
index a5beed1d..52da2b09 100644
--- a/easy-es-common/src/main/java/org/dromara/easyes/common/utils/jackson/JsonUtils.java
+++ b/easy-es-common/src/main/java/org/dromara/easyes/common/utils/jackson/JsonUtils.java
@@ -35,6 +35,7 @@ public class JsonUtils {
// 反序列化时是否将一个对象封装成单元素数组
.configure(DeserializationFeature.ACCEPT_SINGLE_VALUE_AS_ARRAY, true)
.configure(SerializationFeature.INDENT_OUTPUT, false)
+ .configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false)
.build();
AnnotationIntrospector anno = base.getSerializerProviderInstance().getAnnotationIntrospector();
base.setAnnotationIntrospector(new AnnotationIntrospectorPair(anno, new JacksonCustomAnnotationIntrospector()));
diff --git a/easy-es-core/src/main/java/org/dromara/easyes/core/conditions/function/Func.java b/easy-es-core/src/main/java/org/dromara/easyes/core/conditions/function/Func.java
index 99606f41..7917ff4e 100644
--- a/easy-es-core/src/main/java/org/dromara/easyes/core/conditions/function/Func.java
+++ b/easy-es-core/src/main/java/org/dromara/easyes/core/conditions/function/Func.java
@@ -1666,21 +1666,21 @@ public interface Func extends Serializable {
/**
* 用户自定义SearchRequest.Builder 用于混合查询
*
- * @param searchSourceBuilder 用户自定义的SearchSourceBuilder
+ * @param searchBuilder 用户自定义的SearchSourceBuilder
* @return wrapper
*/
- default Children setSearchSourceBuilder(SearchRequest.Builder searchSourceBuilder) {
- return setSearchSourceBuilder(true, searchSourceBuilder);
+ default Children setSearchBuilder(SearchRequest.Builder searchBuilder) {
+ return setSearchBuilder(true, searchBuilder);
}
/**
* 用户自定义SearchRequest.Builder 用于混合查询
*
- * @param condition 执行条件
- * @param searchSourceBuilder 用户自定义的SearchSourceBuilder
+ * @param condition 执行条件
+ * @param searchBuilder 用户自定义的SearchSourceBuilder
* @return wrapper
*/
- Children setSearchSourceBuilder(boolean condition, SearchRequest.Builder searchSourceBuilder);
+ Children setSearchBuilder(boolean condition, SearchRequest.Builder searchBuilder);
/**
* 混合查询
diff --git a/easy-es-core/src/main/java/org/dromara/easyes/core/kernel/AbstractChainWrapper.java b/easy-es-core/src/main/java/org/dromara/easyes/core/kernel/AbstractChainWrapper.java
index bbbb109b..2bd4f715 100644
--- a/easy-es-core/src/main/java/org/dromara/easyes/core/kernel/AbstractChainWrapper.java
+++ b/easy-es-core/src/main/java/org/dromara/easyes/core/kernel/AbstractChainWrapper.java
@@ -657,8 +657,8 @@ public abstract class AbstractChainWrapper {
*/
Integer updateBatchByIds(String routing, Collection entityList, String... indexNames);
+ /**
+ * 根据 whereEntity 条件,更新记录
+ *
+ * @param updateWrapper 条件
+ * @return 成功条数
+ */
+ default Integer update(Wrapper updateWrapper) {
+ return update(null, updateWrapper);
+ }
+
/**
* 根据 whereEntity 条件,更新记录
*
diff --git a/easy-es-core/src/main/java/org/dromara/easyes/core/kernel/BaseEsMapperImpl.java b/easy-es-core/src/main/java/org/dromara/easyes/core/kernel/BaseEsMapperImpl.java
index 2863da81..94fa5ba5 100644
--- a/easy-es-core/src/main/java/org/dromara/easyes/core/kernel/BaseEsMapperImpl.java
+++ b/easy-es-core/src/main/java/org/dromara/easyes/core/kernel/BaseEsMapperImpl.java
@@ -23,13 +23,12 @@ import co.elastic.clients.elasticsearch.indices.RefreshResponse;
import co.elastic.clients.elasticsearch.sql.QueryResponse;
import co.elastic.clients.elasticsearch.sql.query.SqlFormat;
import co.elastic.clients.json.JsonData;
+import co.elastic.clients.json.JsonpUtils;
import co.elastic.clients.transport.TransportOptions;
import co.elastic.clients.transport.rest_client.RestClientTransport;
import com.alibaba.fastjson.JSON;
import lombok.Setter;
import lombok.SneakyThrows;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.http.util.EntityUtils;
import org.dromara.easyes.annotation.rely.IdType;
import org.dromara.easyes.annotation.rely.RefreshPolicy;
@@ -69,17 +68,15 @@ import static org.dromara.easyes.core.kernel.WrapperProcessor.getIndexName;
*
* Copyright © 2021 xpc1024 All Rights Reserved
**/
+@Setter
public class BaseEsMapperImpl implements BaseEsMapper {
- private static final Log log = LogFactory.getLog(BaseEsMapperImpl.class);
/**
* restHighLevel client
*/
- @Setter
private ElasticsearchClient client;
/**
* T 对应的类
*/
- @Setter
private Class entityClass;
@Override
@@ -169,7 +166,7 @@ public class BaseEsMapperImpl implements BaseEsMapper {
@SneakyThrows
public String executeSQL(String sql) {
QueryResponse response = client.sql().query(x -> x.query(sql).format(SqlFormat.Json));
- return response.toString();
+ return JsonpUtils.toString(response, new StringBuilder()).toString();
}
/**
@@ -205,7 +202,7 @@ public class BaseEsMapperImpl implements BaseEsMapper {
@Override
public SearchRequest.Builder getSearchSourceBuilder(Wrapper wrapper) {
- return WrapperProcessor.buildSearchSourceBuilder(wrapper, entityClass);
+ return WrapperProcessor.buildSearchBuilder(wrapper, entityClass);
}
@Override
@@ -213,7 +210,11 @@ public class BaseEsMapperImpl implements BaseEsMapper {
try {
// 用户在wrapper中指定的混合查询条件优先级最高
SearchRequest.Builder builder = Optional.ofNullable(wrapper.searchBuilder)
- .orElse(WrapperProcessor.buildSearchSourceBuilder(wrapper, entityClass));
+ .orElse(WrapperProcessor.buildSearchBuilder(wrapper, entityClass))
+ .index(WrapperProcessor.getIndexName(entityClass, wrapper.indexNames))
+ .routing(wrapper.routing)
+ .preference(wrapper.preference)
+ ;
return builder.build().toString();
} catch (Exception e) {
throw ExceptionUtils.eee("get search source exception", e);
@@ -751,8 +752,6 @@ public class BaseEsMapperImpl implements BaseEsMapper {
);
// 执行批量请求并返回结果
- PrintUtils.printDsl(bulkRequest, client);
-
return doBulkRequest(bulkRequest, Result.Created.jsonValue());
}
@@ -844,7 +843,6 @@ public class BaseEsMapperImpl implements BaseEsMapper {
.operations(operations)
);
- PrintUtils.printDsl(request, client);
return doBulkRequest(request, Result.Deleted.jsonValue());
}
@@ -950,7 +948,6 @@ public class BaseEsMapperImpl implements BaseEsMapper {
.operations(operations)
);
- PrintUtils.printDsl(request, client);
return doBulkRequest(request, Result.Updated.jsonValue());
}
@@ -1037,7 +1034,11 @@ public class BaseEsMapperImpl implements BaseEsMapper {
private SearchResponse getSearchResponse(Wrapper wrapper, List searchAfter, boolean needSearchAfter) {
// 用户在wrapper中指定的混合查询条件优先级最高
SearchRequest.Builder builder = Optional.ofNullable(wrapper.searchBuilder)
- .orElse(WrapperProcessor.buildSearchSourceBuilder(wrapper, entityClass));
+ .orElse(WrapperProcessor.buildSearchBuilder(wrapper, entityClass))
+ .index(WrapperProcessor.getIndexName(entityClass, wrapper.indexNames))
+ .routing(wrapper.routing)
+ .preference(wrapper.preference)
+ ;
if (needSearchAfter && CollectionUtils.isNotEmpty(searchAfter)) {
builder.searchAfter(searchAfter);
@@ -1466,6 +1467,7 @@ public class BaseEsMapperImpl implements BaseEsMapper {
* @return 成功个数
*/
private Integer doBulkRequest(BulkRequest bulkRequest, String successResult) {
+ PrintUtils.printDsl(bulkRequest, client);
try {
BulkResponse bulkResponse = client.withTransportOptions(getTransportOptions()).bulk(bulkRequest);
if (bulkResponse.errors()) {
@@ -1490,7 +1492,11 @@ public class BaseEsMapperImpl implements BaseEsMapper {
private List> getSearchHits(Wrapper wrapper) {
// 用户在wrapper中指定的混合查询条件优先级最高
SearchRequest searchRequest = Optional.ofNullable(wrapper.searchBuilder)
- .orElse(WrapperProcessor.buildSearchSourceBuilder(wrapper, entityClass)).build();
+ .orElse(WrapperProcessor.buildSearchBuilder(wrapper, entityClass))
+ .index(WrapperProcessor.getIndexName(entityClass, wrapper.indexNames))
+ .routing(wrapper.routing)
+ .preference(wrapper.preference)
+ .build();
PrintUtils.printDsl(searchRequest, client);
SearchResponse response;
diff --git a/easy-es-core/src/main/java/org/dromara/easyes/core/kernel/WrapperProcessor.java b/easy-es-core/src/main/java/org/dromara/easyes/core/kernel/WrapperProcessor.java
index b8e1f154..020d9409 100644
--- a/easy-es-core/src/main/java/org/dromara/easyes/core/kernel/WrapperProcessor.java
+++ b/easy-es-core/src/main/java/org/dromara/easyes/core/kernel/WrapperProcessor.java
@@ -17,6 +17,7 @@ import org.dromara.easyes.core.biz.*;
import org.dromara.easyes.core.cache.GlobalConfigCache;
import org.dromara.easyes.core.toolkit.EntityInfoHelper;
import org.dromara.easyes.core.toolkit.FieldUtils;
+import org.dromara.easyes.core.toolkit.GeoUtils;
import org.dromara.easyes.core.toolkit.TreeBuilder;
import org.elasticsearch.geometry.Geometry;
@@ -43,12 +44,12 @@ public class WrapperProcessor {
* @param entityClass 实体类
* @return ES查询参数
*/
- public static SearchRequest.Builder buildSearchSourceBuilder(Wrapper> wrapper, Class> entityClass) {
+ public static SearchRequest.Builder buildSearchBuilder(Wrapper> wrapper, Class> entityClass) {
// 初始化boolQueryBuilder 参数
BoolQuery.Builder boolQueryBuilder = initBoolQueryBuilder(wrapper.paramQueue, entityClass);
// 初始化searchSourceBuilder 参数
- SearchRequest.Builder searchSourceBuilder = initSearchSourceBuilder(wrapper, entityClass);
+ SearchRequest.Builder searchSourceBuilder = initSearchBuilder(wrapper, entityClass);
// 设置boolQuery参数
searchSourceBuilder.query(x -> x.bool(boolQueryBuilder.build()));
@@ -191,8 +192,8 @@ public class WrapperProcessor {
query = Query.of(q -> q.range(p ->
p.untyped(v -> {
v.field(realField).gte(JsonData.of(param.getExt1())).lte(JsonData.of(param.getExt2())).boost(param.getBoost());
- Optional.ofNullable(param.getExt1()).ifPresent(ext1 -> v.timeZone(((ZoneId) ext1).getId()));
- Optional.ofNullable(param.getExt2()).ifPresent(ext2 -> v.format(ext2.toString()));
+ Optional.ofNullable(param.getExt3()).ifPresent(ex -> v.timeZone(((ZoneId) ex).getId()));
+ Optional.ofNullable(param.getExt4()).ifPresent(ex -> v.format(ex.toString()));
return v;
})
));
@@ -228,11 +229,10 @@ public class WrapperProcessor {
case GEO_DISTANCE:
realField = getRealField(param.getColumn(), mappingColumnMap);
query = Query.of(q -> q.geoDistance(p -> {
- p.field(realField).location(x -> x.latlon((LatLonGeoLocation) param.getExt2())).boost(param.getBoost());
+ p.field(realField).location((GeoLocation) param.getExt2()).boost(param.getBoost());
String unit = param.getExt1() == null ? DistanceUnit.Meters.jsonValue() : ((DistanceUnit) param.getExt1()).jsonValue();
- String val = (String) param.getVal();
- String distance = val.endsWith(unit) ? val : val + unit;
- p.distance(distance);
+ Double distance = (Double) param.getVal();
+ p.distance(distance + unit);
return p;
}));
setBool(bool, query, param.getPrevQueryType());
@@ -240,7 +240,7 @@ public class WrapperProcessor {
case GEO_POLYGON:
realField = getRealField(param.getColumn(), mappingColumnMap);
query = Query.of(q -> q.geoPolygon(p -> p.field(realField).polygon(x ->
- x.points((List) param.getExt2()))));
+ x.points((List) param.getVal()))));
setBool(bool, query, param.getPrevQueryType());
break;
case GEO_SHAPE_ID:
@@ -251,10 +251,15 @@ public class WrapperProcessor {
break;
case GEO_SHAPE:
realField = getRealField(param.getColumn(), mappingColumnMap);
- query = Query.of(q -> q.geoShape(p -> p.field(realField).shape(x ->
- x.shape(JsonData.of((Geometry)param.getVal())).relation((GeoShapeRelation) param.getExt1())
- ).boost(param.getBoost())
- ));
+ query = QueryBuilders.geoShape()
+ .field(realField)
+ .shape(x -> x
+// .shape(JsonData.of(WellKnownText.toWKT(val)))
+ .shape(JsonData.of(GeoUtils.toMap((Geometry) param.getVal())))
+ .relation((GeoShapeRelation) param.getExt1())
+ )
+ .boost(param.getBoost())
+ .build()._toQuery();
setBool(bool, query, param.getPrevQueryType());
break;
case PARENT_ID:
@@ -417,17 +422,13 @@ public class WrapperProcessor {
* @param wrapper 条件
* @return SearchSourceBuilder
*/
- private static SearchRequest.Builder initSearchSourceBuilder(Wrapper> wrapper, Class> entityClass) {
+ private static SearchRequest.Builder initSearchBuilder(Wrapper> wrapper, Class> entityClass) {
EntityInfo entityInfo = EntityInfoHelper.getEntityInfo(entityClass);
// 获取自定义字段map
Map mappingColumnMap = entityInfo.getMappingColumnMap();
SearchRequest.Builder builder = new SearchRequest.Builder();
- builder.index(getIndexName(entityClass, wrapper.indexNames));
- builder.routing(wrapper.routing);
- builder.preference(wrapper.preference);
-
// 设置高亮
setHighLight(entityInfo.getHighlightParams(), builder);
@@ -666,17 +667,19 @@ public class WrapperProcessor {
Aggregation.Builder.ContainerBuilder cursor = null;
for (AggregationParam aggParam : aggregationParamList) {
String realField = getRealField(aggParam.getField(), mappingColumnMap);
- Aggregation.Builder.ContainerBuilder builder = getRealAggregationBuilder(aggParam.getAggregationType()
- , aggParam.getName(), realField, wrapper.size, wrapper.bucketOrders);
+ Aggregation.Builder.ContainerBuilder builder = getRealAggregationBuilder(
+ aggParam.getAggregationType(), realField, wrapper.size, wrapper.bucketOrders);
+ // 解决同一个字段聚合多次,如min(starNum), max(starNum) 字段名重复问题
+ String aggName = aggParam.getName() + aggParam.getAggregationType().getValue();
if (aggParam.isEnablePipeline()) {
// 管道聚合, 构造聚合树
if (root == null) {
root = builder;
- rootName = aggParam.getName();
+ rootName = aggName;
cursor = root;
} else {
Aggregation agg = builder.build();
- cursor.aggregations(aggParam.getName(), agg);
+ cursor.aggregations(aggName, agg);
// 解决max、min、avg和sum聚合函数不支持sub-aggregations的问题
if (agg._kind().equals(Aggregation.Kind.Terms)) {
cursor = builder;
@@ -685,7 +688,7 @@ public class WrapperProcessor {
} else {
// 非管道聚合
if (builder != null) {
- searchSourceBuilder.aggregations(aggParam.getName(), builder.build());
+ searchSourceBuilder.aggregations(aggName, builder.build());
}
}
@@ -704,38 +707,28 @@ public class WrapperProcessor {
* 根据聚合类型获取具体的聚合建造者
*
* @param aggType 聚合类型
- * @param name 聚合返回桶的名称 保持原字段名称
* @param realField 原字段名称
* @param size 聚合桶大小
* @return 聚合建造者
*/
private static Aggregation.Builder.ContainerBuilder getRealAggregationBuilder(
AggregationTypeEnum aggType,
- String name,
String realField,
Integer size,
List> bucketOrders
) {
// 解决同一个字段聚合多次,如min(starNum), max(starNum) 字段名重复问题
- String finalName = name + aggType.getValue();
- switch (aggType) {
- case AVG:
- return new Aggregation.Builder().avg(x -> x.field(realField));
- case MIN:
- return new Aggregation.Builder().min(x -> x.field(finalName));
- case MAX:
- return new Aggregation.Builder().max(x -> x.field(realField));
- case SUM:
- return new Aggregation.Builder().sum(x -> x.field(realField));
- case TERMS:
- return new Aggregation.Builder().terms(x -> {
- x.field(realField);
- Optional.ofNullable(size).ifPresent(x::size);
- Optional.ofNullable(bucketOrders).ifPresent(x::order);
- return x;
- });
- default:
- throw new UnsupportedOperationException("不支持的聚合类型,参见AggregationTypeEnum");
- }
+ return switch (aggType) {
+ case AVG -> new Aggregation.Builder().avg(x -> x.field(realField));
+ case MIN -> new Aggregation.Builder().min(x -> x.field(realField));
+ case MAX -> new Aggregation.Builder().max(x -> x.field(realField));
+ case SUM -> new Aggregation.Builder().sum(x -> x.field(realField));
+ case TERMS -> new Aggregation.Builder().terms(x -> {
+ x.field(realField);
+ Optional.ofNullable(size).ifPresent(x::size);
+ Optional.ofNullable(bucketOrders).ifPresent(x::order);
+ return x;
+ });
+ };
}
}
diff --git a/easy-es-core/src/main/java/org/dromara/easyes/core/toolkit/GeoUtils.java b/easy-es-core/src/main/java/org/dromara/easyes/core/toolkit/GeoUtils.java
index 71ecb003..8a647d94 100644
--- a/easy-es-core/src/main/java/org/dromara/easyes/core/toolkit/GeoUtils.java
+++ b/easy-es-core/src/main/java/org/dromara/easyes/core/toolkit/GeoUtils.java
@@ -1,17 +1,15 @@
package org.dromara.easyes.core.toolkit;
+import co.elastic.clients.elasticsearch._types.DistanceUnit;
import co.elastic.clients.elasticsearch._types.GeoLocation;
import org.dromara.easyes.common.exception.EasyEsException;
-import org.elasticsearch.geometry.Geometry;
-import org.elasticsearch.geometry.Point;
-import org.elasticsearch.geometry.Rectangle;
-import org.elasticsearch.geometry.ShapeType;
+import org.elasticsearch.geometry.*;
import org.elasticsearch.geometry.utils.BitUtil;
import org.elasticsearch.geometry.utils.GeographyValidator;
import org.elasticsearch.geometry.utils.Geohash;
import org.elasticsearch.geometry.utils.WellKnownText;
-import java.util.Locale;
+import java.util.*;
/**
* geo工具类
@@ -122,4 +120,204 @@ public class GeoUtils {
BOTTOM_LEFT,
BOTTOM_RIGHT
}
+
+ /**
+ * geoJson 转换
+ *
+ * @param geometry geometry
+ * @return Map
+ */
+ public static Map toMap(Geometry geometry) {
+ Map root = new HashMap<>();
+ root.put("type", getGeoJsonName(geometry));
+
+ geometry.visit(new GeometryVisitor() {
+ @Override
+ public Void visit(Circle circle) {
+ root.put("radius", circle.getRadiusMeters() + DistanceUnit.Meters.jsonValue());
+ root.put("coordinates", coordinatesToList(circle.getY(), circle.getX(), circle.getZ()));
+ return null;
+ }
+
+ @Override
+ public Void visit(GeometryCollection> collection) {
+ List