From ba7708db6166d74a8b26e855eca57f8f45a664d5 Mon Sep 17 00:00:00 2001 From: xpc1024 <252645816@qq.com> Date: Fri, 19 Aug 2022 11:12:52 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E5=AF=B9text=E7=B1=BB?= =?UTF-8?q?=E5=9E=8BfieldData=E8=AE=BE=E7=BD=AE,=E8=AE=BE=E7=BD=AE?= =?UTF-8?q?=E5=90=8E=E5=8F=AF=E6=94=AF=E6=8C=81text=E7=B1=BB=E5=9E=8B?= =?UTF-8?q?=E5=AD=97=E6=AE=B5=E7=9A=84=E8=81=9A=E5=90=88=20=E6=AD=A4?= =?UTF-8?q?=E5=8A=9F=E8=83=BD=E6=9C=89Roin,=E7=A0=81=E4=BA=91ID:=E8=8A=B1?= =?UTF-8?q?=E8=90=BD=E9=99=8C=E8=B4=A1=E7=8C=AE=20=E4=BB=A3=E7=A0=81?= =?UTF-8?q?=E5=86=B2=E7=AA=81=E5=92=8C=E5=8F=911.0=E7=89=88=E6=9C=AC?= =?UTF-8?q?=E6=B1=82=E7=A8=B3=E7=AD=89=E5=8E=9F=E5=9B=A0,=E6=9A=82?= =?UTF-8?q?=E7=94=B1=E4=BD=9C=E8=80=85=E4=BB=A3=E6=8F=90=E4=BA=A4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../java/cn/easyes/annotation/IndexField.java | 7 ++ .../common/constants/BaseEsConstants.java | 4 + .../cn/easyes/core/biz/EntityFieldInfo.java | 4 + .../java/cn/easyes/core/biz/EsIndexParam.java | 4 + .../core/conditions/LambdaEsIndexWrapper.java | 7 +- .../core/conditions/interfaces/Index.java | 48 ++++++---- .../easyes/core/toolkit/EntityInfoHelper.java | 2 + .../cn/easyes/core/toolkit/IndexUtils.java | 12 ++- .../java/cn/easyes/test/entity/Document.java | 5 ++ .../cn/easyes/test/other/FieldDataTest.java | 87 +++++++++++++++++++ 10 files changed, 159 insertions(+), 21 deletions(-) create mode 100644 easy-es-test/src/test/java/cn/easyes/test/other/FieldDataTest.java diff --git a/easy-es-annotation/src/main/java/cn/easyes/annotation/IndexField.java b/easy-es-annotation/src/main/java/cn/easyes/annotation/IndexField.java index 5d19a969..3894f5cc 100644 --- a/easy-es-annotation/src/main/java/cn/easyes/annotation/IndexField.java +++ b/easy-es-annotation/src/main/java/cn/easyes/annotation/IndexField.java @@ -40,6 +40,13 @@ public @interface IndexField { */ FieldType fieldType() default FieldType.NONE; + /** + * 设置text、keyword_text 可以进行聚合操作 + * + * @return 是否设置可聚合 + */ + boolean fieldData() default false; + /** * 索引文档时用的分词器 * diff --git a/easy-es-common/src/main/java/cn/easyes/common/constants/BaseEsConstants.java b/easy-es-common/src/main/java/cn/easyes/common/constants/BaseEsConstants.java index 87e371f1..534506a9 100644 --- a/easy-es-common/src/main/java/cn/easyes/common/constants/BaseEsConstants.java +++ b/easy-es-common/src/main/java/cn/easyes/common/constants/BaseEsConstants.java @@ -226,5 +226,9 @@ public interface BaseEsConstants { * 高亮截取默认长度 */ int DEFAULT_FRAGMENT_SIZE = 100; + /** + * 针对text进行聚合 + */ + String FIELD_DATA = "fielddata"; } diff --git a/easy-es-core/src/main/java/cn/easyes/core/biz/EntityFieldInfo.java b/easy-es-core/src/main/java/cn/easyes/core/biz/EntityFieldInfo.java index b5297fe5..91eac6d9 100644 --- a/easy-es-core/src/main/java/cn/easyes/core/biz/EntityFieldInfo.java +++ b/easy-es-core/src/main/java/cn/easyes/core/biz/EntityFieldInfo.java @@ -39,6 +39,10 @@ public class EntityFieldInfo { * 自动在es中的存储类型 */ private FieldType fieldType; + /** + * 设置text、keyword_text 可以进行聚合操作 + */ + private boolean fieldData; /** * 分词器 */ diff --git a/easy-es-core/src/main/java/cn/easyes/core/biz/EsIndexParam.java b/easy-es-core/src/main/java/cn/easyes/core/biz/EsIndexParam.java index dedf3c29..8ad82171 100644 --- a/easy-es-core/src/main/java/cn/easyes/core/biz/EsIndexParam.java +++ b/easy-es-core/src/main/java/cn/easyes/core/biz/EsIndexParam.java @@ -21,6 +21,10 @@ public class EsIndexParam { * 字段类型 */ private String fieldType; + /** + * 对 text 字段进行聚合处理 + */ + private Boolean fieldData; /** * 分词器 */ diff --git a/easy-es-core/src/main/java/cn/easyes/core/conditions/LambdaEsIndexWrapper.java b/easy-es-core/src/main/java/cn/easyes/core/conditions/LambdaEsIndexWrapper.java index cd340a78..4286d6b2 100644 --- a/easy-es-core/src/main/java/cn/easyes/core/conditions/LambdaEsIndexWrapper.java +++ b/easy-es-core/src/main/java/cn/easyes/core/conditions/LambdaEsIndexWrapper.java @@ -110,8 +110,8 @@ public class LambdaEsIndexWrapper extends Wrapper implements Index mapping(String column, FieldType fieldType, String analyzer, String searchAnalyzer, String dateFormat, Float boost) { - addEsIndexParam(column, fieldType, analyzer, analyzer, dateFormat, boost); + public LambdaEsIndexWrapper mapping(String column, FieldType fieldType, String analyzer, String searchAnalyzer, String dateFormat, Boolean fieldData, Float boost) { + addEsIndexParam(column, fieldType, analyzer, analyzer, dateFormat, fieldData, boost); return typedThis; } @@ -138,13 +138,14 @@ public class LambdaEsIndexWrapper extends Wrapper implements Index extends Serializable { default Children mapping(R column, FieldType fieldType) { - return mapping(column, fieldType, null, null, null, null); + return mapping(column, fieldType, null, null, null, null, null); + } + + default Children mapping(R column, FieldType fieldType, Boolean fieldData) { + return mapping(column, fieldType, null, null, null, fieldData, null); } default Children mapping(R column, FieldType fieldType, Float boost) { - return mapping(column, fieldType, null, null, null, boost); + return mapping(column, fieldType, null, null, null, null, boost); + } + + default Children mapping(R column, FieldType fieldType, Boolean fieldData, Float boost) { + return mapping(column, fieldType, null, null, null, fieldData, boost); } default Children mapping(R column, FieldType fieldType, String dateFormat) { - return mapping(column, fieldType, null, null, dateFormat, null); + return mapping(column, fieldType, null, null, dateFormat, null, null); } default Children mapping(R column, FieldType fieldType, String analyzer, String searchAnalyzer) { - return mapping(column, fieldType, analyzer, searchAnalyzer, null, null); + return mapping(column, fieldType, analyzer, searchAnalyzer, null, null, null); } default Children mapping(R column, FieldType fieldType, String analyzer, String searchAnalyzer, String dateFormat) { - return mapping(column, fieldType, analyzer, searchAnalyzer, dateFormat, null); + return mapping(column, fieldType, analyzer, searchAnalyzer, dateFormat, null, null); } default Children mapping(R column, FieldType fieldType, String analyzer, String searchAnalyzer, Float boost) { - return mapping(column, fieldType, analyzer, searchAnalyzer, null, boost); + return mapping(column, fieldType, analyzer, searchAnalyzer, null, null, boost); } /** @@ -80,16 +87,21 @@ public interface Index extends Serializable { * @param analyzer 分词器类型 * @param searchAnalyzer 查询分词器类型 * @param dateFormat 日期格式 + * @param fieldData 是否支持text字段聚合 * @param boost 权重值 * @return 泛型 */ - default Children mapping(R column, FieldType fieldType, String analyzer, String searchAnalyzer, String dateFormat, Float boost) { - return mapping(FieldUtils.getFieldName(column), fieldType, analyzer, searchAnalyzer, dateFormat, boost); + default Children mapping(R column, FieldType fieldType, String analyzer, String searchAnalyzer, String dateFormat, Boolean fieldData, Float boost) { + return mapping(FieldUtils.getFieldName(column), fieldType, analyzer, searchAnalyzer, dateFormat, fieldData, boost); } default Children mapping(String column, FieldType fieldType) { - return mapping(column, fieldType, null, null, null, null); + return mapping(column, fieldType, null, null, null); + } + + default Children mapping(String column, FieldType fieldType, Boolean fieldData) { + return mapping(column, fieldType, null, null, fieldData, null); } default Children mapping(String column, FieldType fieldType, Float boost) { @@ -98,17 +110,22 @@ public interface Index extends Serializable { default Children mapping(String column, FieldType fieldType, String analyzer) { - return mapping(column, fieldType, analyzer, null, null, null); + return mapping(column, fieldType, analyzer, null, null); } default Children mapping(String column, FieldType fieldType, String analyzer, String searchAnalyzer) { - return mapping(column, fieldType, analyzer, searchAnalyzer, BaseEsConstants.DEFAULT_BOOST); + return mapping(column, fieldType, analyzer, searchAnalyzer, null); } - default Children mapping(String column, FieldType fieldType, String analyzer, String searchAnalyzer, Float boost) { - return mapping(column, fieldType, analyzer, searchAnalyzer, null, boost); + default Children mapping(String column, FieldType fieldType, String analyzer, String searchAnalyzer, Boolean fieldData) { + return mapping(column, fieldType, analyzer, searchAnalyzer, fieldData, null); } + default Children mapping(String column, FieldType fieldType, String analyzer, String searchAnalyzer, Boolean fieldData, Float boost) { + return mapping(column, fieldType, analyzer, searchAnalyzer, null, fieldData, boost); + } + + /** * 设置mapping信息 * @@ -117,10 +134,11 @@ public interface Index extends Serializable { * @param analyzer 分词器类型 * @param searchAnalyzer 查询分词器类型 * @param dateFormat 日期格式 + * @param fieldData 是否支持text字段聚合 * @param boost 字段权重值 * @return 泛型 */ - Children mapping(String column, FieldType fieldType, String analyzer, String searchAnalyzer, String dateFormat, Float boost); + Children mapping(String column, FieldType fieldType, String analyzer, String searchAnalyzer, String dateFormat, Boolean fieldData, Float boost); /** * 设置创建别名信息 diff --git a/easy-es-core/src/main/java/cn/easyes/core/toolkit/EntityInfoHelper.java b/easy-es-core/src/main/java/cn/easyes/core/toolkit/EntityInfoHelper.java index 6ae35771..24addd70 100644 --- a/easy-es-core/src/main/java/cn/easyes/core/toolkit/EntityInfoHelper.java +++ b/easy-es-core/src/main/java/cn/easyes/core/toolkit/EntityInfoHelper.java @@ -320,6 +320,7 @@ public class EntityInfoHelper { entityFieldInfo.setAnalyzer(tableField.analyzer()); entityFieldInfo.setSearchAnalyzer(tableField.searchAnalyzer()); entityFieldInfo.setFieldType(tableField.fieldType()); + entityFieldInfo.setFieldData(tableField.fieldData()); entityFieldInfo.setColumnType(field.getType().getSimpleName()); // 父子类型 @@ -422,6 +423,7 @@ public class EntityInfoHelper { entityFieldInfo.setMappingColumn(mappingColumn); FieldType fieldType = FieldType.NESTED.equals(tableField.fieldType()) ? FieldType.NESTED : FieldType.TEXT; entityFieldInfo.setFieldType(fieldType); + entityFieldInfo.setFieldData(tableField.fieldData()); entityFieldInfo.setColumnType(fieldType.getType()); entityFieldInfo.setAnalyzer(tableField.analyzer()); entityFieldInfo.setSearchAnalyzer(tableField.searchAnalyzer()); diff --git a/easy-es-core/src/main/java/cn/easyes/core/toolkit/IndexUtils.java b/easy-es-core/src/main/java/cn/easyes/core/toolkit/IndexUtils.java index 77004938..fdb7aef3 100644 --- a/easy-es-core/src/main/java/cn/easyes/core/toolkit/IndexUtils.java +++ b/easy-es-core/src/main/java/cn/easyes/core/toolkit/IndexUtils.java @@ -381,16 +381,21 @@ public class IndexUtils { info.put(BaseEsConstants.TYPE, indexParam.getFieldType()); } - // 设置分词器 - boolean needAnalyzer = FieldType.TEXT.getType().equals(indexParam.getFieldType()) || + // 是否text类型或keyword_text类型 + boolean containsTextType = FieldType.TEXT.getType().equals(indexParam.getFieldType()) || FieldType.KEYWORD_TEXT.getType().equals(indexParam.getFieldType()); - if (needAnalyzer) { + if (containsTextType) { + // 设置分词器 Optional.ofNullable(indexParam.getAnalyzer()) .ifPresent(analyzer -> info.put(BaseEsConstants.ANALYZER, indexParam.getAnalyzer().toLowerCase())); Optional.ofNullable(indexParam.getSearchAnalyzer()) .ifPresent(searchAnalyzer -> info.put(BaseEsConstants.SEARCH_ANALYZER, indexParam.getSearchAnalyzer().toLowerCase())); + + // 设置是否对text类型进行聚合处理 + Optional.ofNullable(indexParam.getFieldData()) + .ifPresent(fieldData -> info.put(FIELD_DATA, indexParam.getFieldData())); } // 设置权重 @@ -529,6 +534,7 @@ public class IndexUtils { EsIndexParam esIndexParam = new EsIndexParam(); String esFieldType = IndexUtils.getEsFieldType(field.getFieldType(), field.getColumnType()); esIndexParam.setFieldType(esFieldType); + esIndexParam.setFieldData(field.isFieldData()); esIndexParam.setFieldName(field.getMappingColumn()); esIndexParam.setDateFormat(field.getDateFormat()); if (FieldType.NESTED.equals(field.getFieldType())) { diff --git a/easy-es-test/src/main/java/cn/easyes/test/entity/Document.java b/easy-es-test/src/main/java/cn/easyes/test/entity/Document.java index 729904d0..a331b9fc 100644 --- a/easy-es-test/src/main/java/cn/easyes/test/entity/Document.java +++ b/easy-es-test/src/main/java/cn/easyes/test/entity/Document.java @@ -45,6 +45,11 @@ public class Document { */ @IndexField(strategy = FieldStrategy.NOT_EMPTY, fieldType = FieldType.KEYWORD_TEXT, analyzer = Analyzer.IK_SMART) private String creator; + /** + * 可以聚合的text类型,字段名字随便取,注解中指定fieldData=true后text类型也可以支持聚合 + */ + @IndexField(fieldType = FieldType.TEXT, fieldData = true) + private String filedData; /** * 创建时间 */ diff --git a/easy-es-test/src/test/java/cn/easyes/test/other/FieldDataTest.java b/easy-es-test/src/test/java/cn/easyes/test/other/FieldDataTest.java new file mode 100644 index 00000000..121382fe --- /dev/null +++ b/easy-es-test/src/test/java/cn/easyes/test/other/FieldDataTest.java @@ -0,0 +1,87 @@ +package cn.easyes.test.other; + +import cn.easyes.common.constants.BaseEsConstants; +import cn.easyes.common.enums.FieldType; +import cn.easyes.core.conditions.LambdaEsIndexWrapper; +import cn.easyes.core.conditions.LambdaEsQueryWrapper; +import cn.easyes.core.toolkit.EntityInfoHelper; +import cn.easyes.core.toolkit.EsWrappers; +import cn.easyes.test.TestEasyEsApplication; +import cn.easyes.test.entity.Document; +import cn.easyes.test.mapper.DocumentMapper; +import org.elasticsearch.geometry.Rectangle; +import org.junit.jupiter.api.*; +import org.springframework.boot.test.context.SpringBootTest; + +import javax.annotation.Resource; +import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; +import java.util.List; + +/** + * fieldData 创建索引测试 + * + * @author dys + * @since 1.0 + */ +@DisplayName("fieldData测试") +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@SpringBootTest(classes = TestEasyEsApplication.class) +public class FieldDataTest { + @Resource + private DocumentMapper documentMapper; + + /** + * 测试自动创建索引并插入数据 + */ + @Test + @Order(1) + void testAutoCreateIndex() { + // 测试插入数据 + Document document = new Document(); + document.setEsId("1"); + document.setTitle("测试文档1"); + document.setContent("测试内容1"); + document.setCreator("老汉1"); + document.setLocation("40.171975,116.587105"); + document.setGmtCreate(LocalDateTime.now().format(DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss"))); + document.setCustomField("自定义字段1"); + document.setNullField("id为1的数据不是null,除此之外其它都是"); + Rectangle rectangle = new Rectangle(39.084509D, 41.187328D, 70.610461D, 20.498353D); + document.setGeoLocation(rectangle.toString()); + document.setStarNum(1); + document.setFiledData("123"); + int successCount = documentMapper.insert(document); + Assertions.assertEquals(successCount, 1); + LambdaEsQueryWrapper lambdaEsQueryWrapper = EsWrappers.lambdaQuery(Document.class) + .matchAllQuery() + // 设置为如果filedData为false 或不设置 text类型排序会报错 + .orderByDesc(Document::getFiledData); + List documents = documentMapper.selectList(lambdaEsQueryWrapper); + Assertions.assertEquals(documents.size(), 1); + } + + /** + * 测试手动创建索引配置是否生效 + */ + @Test + @Order(2) + void testFiledData() { + LambdaEsIndexWrapper wrapper = EsWrappers.lambdaIndex(Document.class) + .indexName("document1") + .mapping(Document::getTitle, FieldType.TEXT, true); + Boolean index = documentMapper.createIndex(wrapper); + Assertions.assertTrue(index); + } + + @Test + @Order(3) + public void testDeleteIndex() { + Boolean dcoument1 = documentMapper.deleteIndex("document1"); + boolean deleted = documentMapper.deleteIndex(EntityInfoHelper.getEntityInfo(Document.class).getIndexName()); + boolean lockDeleted = documentMapper.deleteIndex(BaseEsConstants.LOCK_INDEX); + Assertions.assertTrue(dcoument1); + Assertions.assertTrue(deleted); + Assertions.assertTrue(lockDeleted); + } +} \ No newline at end of file