admin/easy-es
1
0
Fork 0
mirror of https://gitee.com/dromara/easy-es.git synced 2025-12-06 17:18:57 +08:00
Code Issues Packages Projects Releases Wiki Activity

!122 feat:mapping支持index和docValues

Browse Source 
Merge pull request !122 from lym/master-index
This commit is contained in:
阿杰 2025-08-18 02:32:07 +00:00 committed by Gitee
commit c13ea2ebc1
No known key found for this signature in database
GPG Key ID: 173E9B9CA92EEF8F
7 changed files with 154 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
easy-es-annotation/src/main/java/org/dromara/easyes/annotation/IndexField.java
View File

@ -116,4 +116,16 @@ public @interface IndexField {
* @return 默认嵌套类
*/
Class<?> nestedOrObjectClass() default DefaultNestedOrObjectClass.class;
/**
* false时禁用倒排索引，无法搜索
* @return 是否启用
*/
boolean index() default true;
/**
* false时禁用列式存储，不可排序、聚合或脚本访问
* @return 是否启用
*/
boolean docValues() default true;
}

9
easy-es-common/src/main/java/org/dromara/easyes/common/utils/MyOptional.java
View File

@ -51,6 +51,15 @@ public final class MyOptional<T> {
}
}
public void ifFalse(Consumer<? super T> consumer) {
if (value != null && value instanceof Boolean) {
boolean condition = (boolean) (Object) value;
if (!condition) {
consumer.accept(value);
}
}
}
public void ifPresent(Consumer<? super T> present, Supplier<?> other) {
if (value != null) {
present.accept(value);

8
easy-es-core/src/main/java/org/dromara/easyes/core/biz/EntityFieldInfo.java
View File

@ -107,6 +107,10 @@ public class EntityFieldInfo {
*/
private List<InnerFieldInfo> innerFieldInfoList;
private Boolean index;
private Boolean docValues;
/**
* 存在 TableField 注解时, 使用的构造函数
*
@ -122,6 +126,8 @@ public class EntityFieldInfo {
} else {
this.fieldStrategy = tableField.strategy();
}
this.index = tableField.index();
this.docValues = tableField.docValues();
}
/**
@ -133,6 +139,8 @@ public class EntityFieldInfo {
public EntityFieldInfo(GlobalConfig.DbConfig dbConfig, Field field) {
this.fieldStrategy = dbConfig.getFieldStrategy();
this.column = field.getName();
this.index = Boolean.TRUE;
this.docValues = Boolean.TRUE;
}
/**

4
easy-es-core/src/main/java/org/dromara/easyes/core/biz/EsIndexParam.java
View File

@ -73,6 +73,10 @@ public class EsIndexParam {
*/
private List<InnerFieldParam> innerFieldParamList;
private Boolean index;
private Boolean docValues;
/**
* 内部段参数
*/

54
easy-es-core/src/main/java/org/dromara/easyes/core/toolkit/IndexUtils.java
View File

@ -455,6 +455,10 @@ public class IndexUtils {
ByteNumberProperty property = ByteNumberProperty.of(a -> {
buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
buildInnerFields(a, indexParam);
// index和docValue为true时，在esMapping中默认不显示，如果这里直接设置为true，会导致
// org.dromara.easyes.core.toolkit.PropertyComparator.isPropertyMapEqual返回false
MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
return a;
});
properties.put(fieldName, property._toProperty());
@ -464,6 +468,8 @@ public class IndexUtils {
ShortNumberProperty property = ShortNumberProperty.of(a -> {
buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
buildInnerFields(a, indexParam);
MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
return a;
});
properties.put(fieldName, property._toProperty());
@ -473,6 +479,8 @@ public class IndexUtils {
IntegerNumberProperty property = IntegerNumberProperty.of(a -> {
buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
buildInnerFields(a, indexParam);
MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
return a;
});
properties.put(fieldName, property._toProperty());
@ -482,6 +490,8 @@ public class IndexUtils {
LongNumberProperty property = LongNumberProperty.of(a -> {
buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
buildInnerFields(a, indexParam);
MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
return a;
});
properties.put(fieldName, property._toProperty());
@ -491,6 +501,8 @@ public class IndexUtils {
FloatNumberProperty property = FloatNumberProperty.of(a -> {
buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
buildInnerFields(a, indexParam);
MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
return a;
});
properties.put(fieldName, property._toProperty());
@ -500,6 +512,8 @@ public class IndexUtils {
DoubleNumberProperty property = DoubleNumberProperty.of(a -> {
buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
buildInnerFields(a, indexParam);
MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
return a;
});
properties.put(fieldName, property._toProperty());
@ -509,6 +523,8 @@ public class IndexUtils {
HalfFloatNumberProperty property = HalfFloatNumberProperty.of(a -> {
buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
buildInnerFields(a, indexParam);
MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
return a;
});
properties.put(fieldName, property._toProperty());
@ -521,6 +537,8 @@ public class IndexUtils {
buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
buildInnerFields(a, indexParam);
a.scalingFactor(scalingFactor);
MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
return a;
});
properties.put(fieldName, property._toProperty());
@ -531,6 +549,8 @@ public class IndexUtils {
a.boost(indexParam.getBoost());
buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
buildInnerFields(a, indexParam);
MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
return a;
});
properties.put(fieldName, property._toProperty());
@ -538,10 +558,11 @@ public class IndexUtils {
}
if (FieldType.DATE.getType().equals(indexParam.getFieldType())) {
DateProperty property = DateProperty.of(a -> {
a.boost(indexParam.getBoost());
a.boost(indexParam.getBoost()).format(indexParam.getDateFormat());
buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
buildInnerFields(a, indexParam);
a.format(indexParam.getDateFormat());
MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
return a;
});
properties.put(fieldName, property._toProperty());
@ -551,6 +572,7 @@ public class IndexUtils {
BinaryProperty property = BinaryProperty.of(a -> {
buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
buildInnerFields(a, indexParam);
MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
return a;
});
properties.put(fieldName, property._toProperty());
@ -558,22 +580,23 @@ public class IndexUtils {
}
if (FieldType.KEYWORD.getType().equals(indexParam.getFieldType())) {
KeywordProperty property = KeywordProperty.of(a -> {
a.boost(indexParam.getBoost());
a.boost(indexParam.getBoost())
.normalizer(indexParam.isIgnoreCase() ? LOWERCASE_NORMALIZER : null);
buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
buildInnerFields(a, indexParam);
a.normalizer(indexParam.isIgnoreCase() ? LOWERCASE_NORMALIZER : null);
MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
return a;
});
properties.put(fieldName, property._toProperty());
return;
}
if (FieldType.TEXT.getType().equals(indexParam.getFieldType())) {
int ignoreAbove = Optional.ofNullable(indexParam.getIgnoreAbove()).orElse(DEFAULT_IGNORE_ABOVE);
TextProperty property = TextProperty.of(a -> {
a.boost(indexParam.getBoost());
a.boost(indexParam.getBoost());
buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
buildInnerFields(a, indexParam);
MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
Optional.ofNullable(indexParam.getAnalyzer()).map(String::toLowerCase).ifPresent(a::analyzer);
Optional.ofNullable(indexParam.getSearchAnalyzer()).map(String::toLowerCase).ifPresent(a::searchAnalyzer);
MyOptional.ofNullable(indexParam.getFieldData()).ifTrue(a::fielddata);
@ -588,9 +611,11 @@ public class IndexUtils {
a.boost(indexParam.getBoost());
buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
buildInnerFields(a, indexParam);
MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
a.fields(FieldType.KEYWORD.getType(), c -> c
.keyword(d -> {
d.ignoreAbove(ignoreAbove);
MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
if (indexParam.isIgnoreCase()) {
d.normalizer(LOWERCASE_NORMALIZER);
}
@ -610,6 +635,7 @@ public class IndexUtils {
WildcardProperty property = WildcardProperty.of(a -> {
buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
buildInnerFields(a, indexParam);
MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
return a;
});
properties.put(fieldName, property._toProperty());
@ -650,6 +676,8 @@ public class IndexUtils {
GeoPointProperty property = GeoPointProperty.of(a -> {
buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
buildInnerFields(a, indexParam);
MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
return a;
});
properties.put(fieldName, property._toProperty());
@ -659,6 +687,7 @@ public class IndexUtils {
GeoShapeProperty property = GeoShapeProperty.of(a -> {
buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
buildInnerFields(a, indexParam);
MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
return a;
});
properties.put(fieldName, property._toProperty());
@ -669,6 +698,8 @@ public class IndexUtils {
a.boost(indexParam.getBoost());
buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
buildInnerFields(a, indexParam);
MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
return a;
});
properties.put(fieldName, property._toProperty());
@ -678,6 +709,7 @@ public class IndexUtils {
CompletionProperty property = CompletionProperty.of(a -> {
buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
buildInnerFields(a, indexParam);
MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
Optional.ofNullable(indexParam.getAnalyzer()).map(String::toLowerCase).ifPresent(a::analyzer);
Optional.ofNullable(indexParam.getSearchAnalyzer()).map(String::toLowerCase).ifPresent(a::searchAnalyzer);
return a;
@ -690,6 +722,8 @@ public class IndexUtils {
a.boost(indexParam.getBoost());
buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
buildInnerFields(a, indexParam);
MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
return a;
});
properties.put(fieldName, property._toProperty());
@ -710,6 +744,7 @@ public class IndexUtils {
DenseVectorProperty property = DenseVectorProperty.of(a -> {
Optional.ofNullable(indexParam.getDims()).ifPresent(a::dims);
buildInnerFields(a, indexParam);
MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
return a;
});
properties.put(fieldName, property._toProperty());
@ -1020,6 +1055,9 @@ public class IndexUtils {
esIndexParam.setScalingFactor(field.getScalingFactor());
esIndexParam.setDims(field.getDims());
esIndexParam.setCopyToList(field.getCopyToList());
esIndexParam.setIgnoreCase(field.isIgnoreCase());
esIndexParam.setIndex(field.getIndex());
esIndexParam.setDocValues(field.getDocValues());
// 嵌套类型
if (FieldType.NESTED.equals(field.getFieldType()) || FieldType.OBJECT.equals(field.getFieldType())) {
@ -1035,8 +1073,6 @@ public class IndexUtils {
esIndexParam.setSearchAnalyzer(field.getSearchAnalyzer());
}
}
esIndexParam.setIgnoreCase(field.isIgnoreCase());
// 日期处理
Optional.ofNullable(dateFormatMap)
.flatMap(i -> Optional.ofNullable(i.get(field.getColumn())))
@ -1093,7 +1129,7 @@ public class IndexUtils {
TypeMapping.Builder builderFromIndex = esIndexInfo.getBuilder();
Map<String, Property> propertiesOfEntity = builderFromEntity.build().properties();
Map<String, Property> propertiesOfIndex = builderFromIndex.build().properties();
return !PropertyComparator.isPropertyMapEqual(propertiesOfEntity,propertiesOfIndex);
return !PropertyComparator.isPropertyMapEqual(propertiesOfEntity, propertiesOfIndex);
}
/**

6
easy-es-springboot-test/src/main/java/org/dromara/easyes/test/entity/Document.java
View File

@ -156,4 +156,10 @@ public class Document extends BaseJoin {
*/
@IndexField(fieldType = FieldType.DENSE_VECTOR, dims = 3)
private double[] vector;
@IndexField(fieldType = FieldType.KEYWORD, index = false)
private String indexFalse;
@IndexField(fieldType = FieldType.KEYWORD, docValues = false)
private String docFalse;
}

70
easy-es-springboot-test/src/test/java/org/dromara/easyes/test/other/IndexFalseTest.java Normal file
View File

@ -0,0 +1,70 @@
package org.dromara.easyes.test.other;
import co.elastic.clients.elasticsearch._types.ElasticsearchException;
import org.dromara.easyes.common.exception.EasyEsException;
import org.dromara.easyes.core.conditions.select.LambdaEsQueryWrapper;
import org.dromara.easyes.test.TestEasyEsApplication;
import org.dromara.easyes.test.entity.Document;
import org.dromara.easyes.test.mapper.DocumentMapper;
import org.junit.jupiter.api.*;
import org.springframework.boot.test.context.SpringBootTest;
import javax.annotation.Resource;
/**
* index和docValues测试
*/
@DisplayName("index和docValues测试")
@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
@SpringBootTest(classes = TestEasyEsApplication.class)
public class IndexFalseTest {
@Resource
private DocumentMapper documentMapper;
/**
* index为false时，无法搜索
*/
@Test
void testIndexFalse() {
// documentMapper.deleteIndex("easyes_document");
// documentMapper.createIndex();
LambdaEsQueryWrapper<Document> wrapper = new LambdaEsQueryWrapper<>();
wrapper.eq(Document::getIndexFalse, "777");
Exception exception = null;
try {
documentMapper.selectList(wrapper);
} catch (ElasticsearchException e) {
e.printStackTrace();
System.out.println(e.response());
// failed to create query: Cannot search on field [indexFalse] since it is not indexed
exception = e;
}
Assertions.assertNotNull(exception);
}
/**
* docValues为false时，无法聚合
*/
@Test
void testDocValueFalse() {
// documentMapper.deleteIndex("easyes_document");
// documentMapper.createIndex();
Exception exception = null;
LambdaEsQueryWrapper<Document> wrapper = new LambdaEsQueryWrapper<>();
wrapper.termsAggregation(Document::getDocFalse);
try {
documentMapper.search(wrapper);
} catch (EasyEsException e) {
e.printStackTrace();
if (e.getCause() instanceof ElasticsearchException) {
// 这个异常描述貌似有问题？
// Can't load fielddata on [docFalse] because fielddata is unsupported on fields of type [keyword]. Use doc values instead.
System.out.println(((ElasticsearchException) e.getCause()).response());
}
exception = e;
}
Assertions.assertNotNull(exception);
}
}