From 3ba4752e694c858e5e3659c85312c9ddcbc163f9 Mon Sep 17 00:00:00 2001 From: lym <2499445495@qq.com> Date: Sat, 19 Jul 2025 09:41:58 +0800 Subject: [PATCH] =?UTF-8?q?feat:mapping=E6=94=AF=E6=8C=81index=E5=92=8Cdoc?= =?UTF-8?q?Values?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../dromara/easyes/annotation/IndexField.java | 12 ++++ .../easyes/common/utils/MyOptional.java | 9 +++ .../easyes/core/biz/EntityFieldInfo.java | 8 +++ .../dromara/easyes/core/biz/EsIndexParam.java | 4 ++ .../easyes/core/toolkit/IndexUtils.java | 54 +++++++++++--- .../dromara/easyes/test/entity/Document.java | 6 ++ .../easyes/test/other/IndexFalseTest.java | 70 +++++++++++++++++++ 7 files changed, 154 insertions(+), 9 deletions(-) create mode 100644 easy-es-springboot-test/src/test/java/org/dromara/easyes/test/other/IndexFalseTest.java diff --git a/easy-es-annotation/src/main/java/org/dromara/easyes/annotation/IndexField.java b/easy-es-annotation/src/main/java/org/dromara/easyes/annotation/IndexField.java index b72b27ac..1e0e2423 100644 --- a/easy-es-annotation/src/main/java/org/dromara/easyes/annotation/IndexField.java +++ b/easy-es-annotation/src/main/java/org/dromara/easyes/annotation/IndexField.java @@ -116,4 +116,16 @@ public @interface IndexField { * @return 默认嵌套类 */ Class nestedOrObjectClass() default DefaultNestedOrObjectClass.class; + + /** + * false时禁用倒排索引，无法搜索 + * @return 是否启用 + */ + boolean index() default true; + + /** + * false时禁用列式存储，不可排序、聚合或脚本访问 + * @return 是否启用 + */ + boolean docValues() default true; } diff --git a/easy-es-common/src/main/java/org/dromara/easyes/common/utils/MyOptional.java b/easy-es-common/src/main/java/org/dromara/easyes/common/utils/MyOptional.java index 7862939f..65634be5 100644 --- a/easy-es-common/src/main/java/org/dromara/easyes/common/utils/MyOptional.java +++ b/easy-es-common/src/main/java/org/dromara/easyes/common/utils/MyOptional.java @@ -51,6 +51,15 @@ public final class MyOptional { } } + public void ifFalse(Consumer consumer) { + if (value != null && value instanceof Boolean) { + boolean condition = (boolean) (Object) value; + if (!condition) { + consumer.accept(value); + } + } + } + public void ifPresent(Consumer present, Supplier other) { if (value != null) { present.accept(value); diff --git a/easy-es-core/src/main/java/org/dromara/easyes/core/biz/EntityFieldInfo.java b/easy-es-core/src/main/java/org/dromara/easyes/core/biz/EntityFieldInfo.java index 8c9f319f..657f4074 100644 --- a/easy-es-core/src/main/java/org/dromara/easyes/core/biz/EntityFieldInfo.java +++ b/easy-es-core/src/main/java/org/dromara/easyes/core/biz/EntityFieldInfo.java @@ -107,6 +107,10 @@ public class EntityFieldInfo { */ private List innerFieldInfoList; + private Boolean index; + + private Boolean docValues; + /** * 存在 TableField 注解时, 使用的构造函数 * @@ -122,6 +126,8 @@ public class EntityFieldInfo { } else { this.fieldStrategy = tableField.strategy(); } + this.index = tableField.index(); + this.docValues = tableField.docValues(); } /** @@ -133,6 +139,8 @@ public class EntityFieldInfo { public EntityFieldInfo(GlobalConfig.DbConfig dbConfig, Field field) { this.fieldStrategy = dbConfig.getFieldStrategy(); this.column = field.getName(); + this.index = Boolean.TRUE; + this.docValues = Boolean.TRUE; } /** diff --git a/easy-es-core/src/main/java/org/dromara/easyes/core/biz/EsIndexParam.java b/easy-es-core/src/main/java/org/dromara/easyes/core/biz/EsIndexParam.java index 653fedcb..64438950 100644 --- a/easy-es-core/src/main/java/org/dromara/easyes/core/biz/EsIndexParam.java +++ b/easy-es-core/src/main/java/org/dromara/easyes/core/biz/EsIndexParam.java @@ -73,6 +73,10 @@ public class EsIndexParam { */ private List innerFieldParamList; + private Boolean index; + + private Boolean docValues; + /** * 内部段参数 */ diff --git a/easy-es-core/src/main/java/org/dromara/easyes/core/toolkit/IndexUtils.java b/easy-es-core/src/main/java/org/dromara/easyes/core/toolkit/IndexUtils.java index e741fcc3..ba4523aa 100644 --- a/easy-es-core/src/main/java/org/dromara/easyes/core/toolkit/IndexUtils.java +++ b/easy-es-core/src/main/java/org/dromara/easyes/core/toolkit/IndexUtils.java @@ -455,6 +455,10 @@ public class IndexUtils { ByteNumberProperty property = ByteNumberProperty.of(a -> { buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList()); buildInnerFields(a, indexParam); + // index和docValue为true时，在esMapping中默认不显示，如果这里直接设置为true，会导致 + // org.dromara.easyes.core.toolkit.PropertyComparator.isPropertyMapEqual返回false + MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index); + MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues); return a; }); properties.put(fieldName, property._toProperty()); @@ -464,6 +468,8 @@ public class IndexUtils { ShortNumberProperty property = ShortNumberProperty.of(a -> { buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList()); buildInnerFields(a, indexParam); + MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index); + MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues); return a; }); properties.put(fieldName, property._toProperty()); @@ -473,6 +479,8 @@ public class IndexUtils { IntegerNumberProperty property = IntegerNumberProperty.of(a -> { buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList()); buildInnerFields(a, indexParam); + MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index); + MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues); return a; }); properties.put(fieldName, property._toProperty()); @@ -482,6 +490,8 @@ public class IndexUtils { LongNumberProperty property = LongNumberProperty.of(a -> { buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList()); buildInnerFields(a, indexParam); + MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index); + MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues); return a; }); properties.put(fieldName, property._toProperty()); @@ -491,6 +501,8 @@ public class IndexUtils { FloatNumberProperty property = FloatNumberProperty.of(a -> { buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList()); buildInnerFields(a, indexParam); + MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index); + MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues); return a; }); properties.put(fieldName, property._toProperty()); @@ -500,6 +512,8 @@ public class IndexUtils { DoubleNumberProperty property = DoubleNumberProperty.of(a -> { buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList()); buildInnerFields(a, indexParam); + MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index); + MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues); return a; }); properties.put(fieldName, property._toProperty()); @@ -509,6 +523,8 @@ public class IndexUtils { HalfFloatNumberProperty property = HalfFloatNumberProperty.of(a -> { buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList()); buildInnerFields(a, indexParam); + MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index); + MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues); return a; }); properties.put(fieldName, property._toProperty()); @@ -521,6 +537,8 @@ public class IndexUtils { buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList()); buildInnerFields(a, indexParam); a.scalingFactor(scalingFactor); + MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index); + MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues); return a; }); properties.put(fieldName, property._toProperty()); @@ -531,6 +549,8 @@ public class IndexUtils { a.boost(indexParam.getBoost()); buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList()); buildInnerFields(a, indexParam); + MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index); + MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues); return a; }); properties.put(fieldName, property._toProperty()); @@ -538,10 +558,11 @@ public class IndexUtils { } if (FieldType.DATE.getType().equals(indexParam.getFieldType())) { DateProperty property = DateProperty.of(a -> { - a.boost(indexParam.getBoost()); + a.boost(indexParam.getBoost()).format(indexParam.getDateFormat()); buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList()); buildInnerFields(a, indexParam); - a.format(indexParam.getDateFormat()); + MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index); + MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues); return a; }); properties.put(fieldName, property._toProperty()); @@ -551,6 +572,7 @@ public class IndexUtils { BinaryProperty property = BinaryProperty.of(a -> { buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList()); buildInnerFields(a, indexParam); + MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues); return a; }); properties.put(fieldName, property._toProperty()); @@ -558,22 +580,23 @@ public class IndexUtils { } if (FieldType.KEYWORD.getType().equals(indexParam.getFieldType())) { KeywordProperty property = KeywordProperty.of(a -> { - a.boost(indexParam.getBoost()); + a.boost(indexParam.getBoost()) + .normalizer(indexParam.isIgnoreCase() ? LOWERCASE_NORMALIZER : null); buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList()); buildInnerFields(a, indexParam); - a.normalizer(indexParam.isIgnoreCase() ? LOWERCASE_NORMALIZER : null); + MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index); + MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues); return a; }); properties.put(fieldName, property._toProperty()); return; } if (FieldType.TEXT.getType().equals(indexParam.getFieldType())) { - int ignoreAbove = Optional.ofNullable(indexParam.getIgnoreAbove()).orElse(DEFAULT_IGNORE_ABOVE); TextProperty property = TextProperty.of(a -> { - a.boost(indexParam.getBoost()); a.boost(indexParam.getBoost()); buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList()); buildInnerFields(a, indexParam); + MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index); Optional.ofNullable(indexParam.getAnalyzer()).map(String::toLowerCase).ifPresent(a::analyzer); Optional.ofNullable(indexParam.getSearchAnalyzer()).map(String::toLowerCase).ifPresent(a::searchAnalyzer); MyOptional.ofNullable(indexParam.getFieldData()).ifTrue(a::fielddata); @@ -588,9 +611,11 @@ public class IndexUtils { a.boost(indexParam.getBoost()); buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList()); buildInnerFields(a, indexParam); + MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index); a.fields(FieldType.KEYWORD.getType(), c -> c .keyword(d -> { d.ignoreAbove(ignoreAbove); + MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index); if (indexParam.isIgnoreCase()) { d.normalizer(LOWERCASE_NORMALIZER); } @@ -610,6 +635,7 @@ public class IndexUtils { WildcardProperty property = WildcardProperty.of(a -> { buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList()); buildInnerFields(a, indexParam); + MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues); return a; }); properties.put(fieldName, property._toProperty()); @@ -650,6 +676,8 @@ public class IndexUtils { GeoPointProperty property = GeoPointProperty.of(a -> { buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList()); buildInnerFields(a, indexParam); + MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index); + MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues); return a; }); properties.put(fieldName, property._toProperty()); @@ -659,6 +687,7 @@ public class IndexUtils { GeoShapeProperty property = GeoShapeProperty.of(a -> { buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList()); buildInnerFields(a, indexParam); + MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues); return a; }); properties.put(fieldName, property._toProperty()); @@ -669,6 +698,8 @@ public class IndexUtils { a.boost(indexParam.getBoost()); buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList()); buildInnerFields(a, indexParam); + MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index); + MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues); return a; }); properties.put(fieldName, property._toProperty()); @@ -678,6 +709,7 @@ public class IndexUtils { CompletionProperty property = CompletionProperty.of(a -> { buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList()); buildInnerFields(a, indexParam); + MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues); Optional.ofNullable(indexParam.getAnalyzer()).map(String::toLowerCase).ifPresent(a::analyzer); Optional.ofNullable(indexParam.getSearchAnalyzer()).map(String::toLowerCase).ifPresent(a::searchAnalyzer); return a; @@ -690,6 +722,8 @@ public class IndexUtils { a.boost(indexParam.getBoost()); buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList()); buildInnerFields(a, indexParam); + MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index); + MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues); return a; }); properties.put(fieldName, property._toProperty()); @@ -710,6 +744,7 @@ public class IndexUtils { DenseVectorProperty property = DenseVectorProperty.of(a -> { Optional.ofNullable(indexParam.getDims()).ifPresent(a::dims); buildInnerFields(a, indexParam); + MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index); return a; }); properties.put(fieldName, property._toProperty()); @@ -1020,6 +1055,9 @@ public class IndexUtils { esIndexParam.setScalingFactor(field.getScalingFactor()); esIndexParam.setDims(field.getDims()); esIndexParam.setCopyToList(field.getCopyToList()); + esIndexParam.setIgnoreCase(field.isIgnoreCase()); + esIndexParam.setIndex(field.getIndex()); + esIndexParam.setDocValues(field.getDocValues()); // 嵌套类型 if (FieldType.NESTED.equals(field.getFieldType()) || FieldType.OBJECT.equals(field.getFieldType())) { @@ -1035,8 +1073,6 @@ public class IndexUtils { esIndexParam.setSearchAnalyzer(field.getSearchAnalyzer()); } } - esIndexParam.setIgnoreCase(field.isIgnoreCase()); - // 日期处理 Optional.ofNullable(dateFormatMap) .flatMap(i -> Optional.ofNullable(i.get(field.getColumn()))) @@ -1093,7 +1129,7 @@ public class IndexUtils { TypeMapping.Builder builderFromIndex = esIndexInfo.getBuilder(); Map propertiesOfEntity = builderFromEntity.build().properties(); Map propertiesOfIndex = builderFromIndex.build().properties(); - return !PropertyComparator.isPropertyMapEqual(propertiesOfEntity,propertiesOfIndex); + return !PropertyComparator.isPropertyMapEqual(propertiesOfEntity, propertiesOfIndex); } /** diff --git a/easy-es-springboot-test/src/main/java/org/dromara/easyes/test/entity/Document.java b/easy-es-springboot-test/src/main/java/org/dromara/easyes/test/entity/Document.java index 73495a4f..97ecbf87 100644 --- a/easy-es-springboot-test/src/main/java/org/dromara/easyes/test/entity/Document.java +++ b/easy-es-springboot-test/src/main/java/org/dromara/easyes/test/entity/Document.java @@ -156,4 +156,10 @@ public class Document extends BaseJoin { */ @IndexField(fieldType = FieldType.DENSE_VECTOR, dims = 3) private double[] vector; + + @IndexField(fieldType = FieldType.KEYWORD, index = false) + private String indexFalse; + + @IndexField(fieldType = FieldType.KEYWORD, docValues = false) + private String docFalse; } diff --git a/easy-es-springboot-test/src/test/java/org/dromara/easyes/test/other/IndexFalseTest.java b/easy-es-springboot-test/src/test/java/org/dromara/easyes/test/other/IndexFalseTest.java new file mode 100644 index 00000000..58f2f225 --- /dev/null +++ b/easy-es-springboot-test/src/test/java/org/dromara/easyes/test/other/IndexFalseTest.java @@ -0,0 +1,70 @@ +package org.dromara.easyes.test.other; + + +import co.elastic.clients.elasticsearch._types.ElasticsearchException; +import org.dromara.easyes.common.exception.EasyEsException; +import org.dromara.easyes.core.conditions.select.LambdaEsQueryWrapper; +import org.dromara.easyes.test.TestEasyEsApplication; +import org.dromara.easyes.test.entity.Document; +import org.dromara.easyes.test.mapper.DocumentMapper; +import org.junit.jupiter.api.*; +import org.springframework.boot.test.context.SpringBootTest; + +import javax.annotation.Resource; + +/** + * index和docValues测试 + */ +@DisplayName("index和docValues测试") +@TestMethodOrder(MethodOrderer.OrderAnnotation.class) +@SpringBootTest(classes = TestEasyEsApplication.class) +public class IndexFalseTest { + @Resource + private DocumentMapper documentMapper; + + /** + * index为false时，无法搜索 + */ + @Test + void testIndexFalse() { +// documentMapper.deleteIndex("easyes_document"); +// documentMapper.createIndex(); + LambdaEsQueryWrapper wrapper = new LambdaEsQueryWrapper<>(); + wrapper.eq(Document::getIndexFalse, "777"); + Exception exception = null; + try { + documentMapper.selectList(wrapper); + } catch (ElasticsearchException e) { + e.printStackTrace(); + System.out.println(e.response()); +// failed to create query: Cannot search on field [indexFalse] since it is not indexed + exception = e; + } + Assertions.assertNotNull(exception); + } + + /** + * docValues为false时，无法聚合 + */ + @Test + void testDocValueFalse() { +// documentMapper.deleteIndex("easyes_document"); +// documentMapper.createIndex(); + Exception exception = null; + LambdaEsQueryWrapper wrapper = new LambdaEsQueryWrapper<>(); + wrapper.termsAggregation(Document::getDocFalse); + try { + documentMapper.search(wrapper); + } catch (EasyEsException e) { + e.printStackTrace(); + if (e.getCause() instanceof ElasticsearchException) { +// 这个异常描述貌似有问题？ +// Can't load fielddata on [docFalse] because fielddata is unsupported on fields of type [keyword]. Use doc values instead. + System.out.println(((ElasticsearchException) e.getCause()).response()); + } + exception = e; + } + Assertions.assertNotNull(exception); + } + +} \ No newline at end of file