!122 feat:mapping支持index和docValues

Merge pull request !122 from lym/master-index

easy-es-annotation/src/main/java/org/dromara/easyes/annotation/IndexField.java

@@ -116,4 +116,16 @@ public @interface IndexField {
      * @return 默认嵌套类
      */
     Class<?> nestedOrObjectClass() default DefaultNestedOrObjectClass.class;
+
+    /**
+     * false时禁用倒排索引，无法搜索
+     * @return 是否启用
+     */
+    boolean index() default true;
+
+    /**
+     * false时禁用列式存储，不可排序、聚合或脚本访问
+     * @return 是否启用
+     */
+    boolean docValues() default true;
 }

easy-es-common/src/main/java/org/dromara/easyes/common/utils/MyOptional.java

@@ -51,6 +51,15 @@ public final class MyOptional<T> {
         }
     }
 
+    public void ifFalse(Consumer<? super T> consumer) {
+        if (value != null && value instanceof Boolean) {
+            boolean condition = (boolean) (Object) value;
+            if (!condition) {
+                consumer.accept(value);
+            }
+        }
+    }
+
     public void ifPresent(Consumer<? super T> present, Supplier<?> other) {
         if (value != null) {
             present.accept(value);

easy-es-core/src/main/java/org/dromara/easyes/core/biz/EntityFieldInfo.java

@@ -107,6 +107,10 @@ public class EntityFieldInfo {
      */
     private List<InnerFieldInfo> innerFieldInfoList;
 
+    private Boolean index;
+
+    private Boolean docValues;
+
     /**
      * 存在 TableField 注解时, 使用的构造函数
      *
@@ -122,6 +126,8 @@ public class EntityFieldInfo {
         } else {
             this.fieldStrategy = tableField.strategy();
         }
+        this.index = tableField.index();
+        this.docValues = tableField.docValues();
     }
 
     /**
@@ -133,6 +139,8 @@ public class EntityFieldInfo {
     public EntityFieldInfo(GlobalConfig.DbConfig dbConfig, Field field) {
         this.fieldStrategy = dbConfig.getFieldStrategy();
         this.column = field.getName();
+        this.index = Boolean.TRUE;
+        this.docValues = Boolean.TRUE;
     }
 
     /**

easy-es-core/src/main/java/org/dromara/easyes/core/biz/EsIndexParam.java

@@ -73,6 +73,10 @@ public class EsIndexParam {
      */
     private List<InnerFieldParam> innerFieldParamList;
 
+    private Boolean index;
+
+    private Boolean docValues;
+
     /**
      * 内部段参数
      */

easy-es-core/src/main/java/org/dromara/easyes/core/toolkit/IndexUtils.java

@@ -455,6 +455,10 @@ public class IndexUtils {
                 ByteNumberProperty property = ByteNumberProperty.of(a -> {
                     buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
                     buildInnerFields(a, indexParam);
+                    // index和docValue为true时，在esMapping中默认不显示，如果这里直接设置为true，会导致
+                    // org.dromara.easyes.core.toolkit.PropertyComparator.isPropertyMapEqual返回false
+                    MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
+                    MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
                     return a;
                 });
                 properties.put(fieldName, property._toProperty());
@@ -464,6 +468,8 @@ public class IndexUtils {
                 ShortNumberProperty property = ShortNumberProperty.of(a -> {
                     buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
                     buildInnerFields(a, indexParam);
+                    MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
+                    MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
                     return a;
                 });
                 properties.put(fieldName, property._toProperty());
@@ -473,6 +479,8 @@ public class IndexUtils {
                 IntegerNumberProperty property = IntegerNumberProperty.of(a -> {
                     buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
                     buildInnerFields(a, indexParam);
+                    MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
+                    MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
                     return a;
                 });
                 properties.put(fieldName, property._toProperty());
@@ -482,6 +490,8 @@ public class IndexUtils {
                 LongNumberProperty property = LongNumberProperty.of(a -> {
                     buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
                     buildInnerFields(a, indexParam);
+                    MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
+                    MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
                     return a;
                 });
                 properties.put(fieldName, property._toProperty());
@@ -491,6 +501,8 @@ public class IndexUtils {
                 FloatNumberProperty property = FloatNumberProperty.of(a -> {
                     buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
                     buildInnerFields(a, indexParam);
+                    MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
+                    MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
                     return a;
                 });
                 properties.put(fieldName, property._toProperty());
@@ -500,6 +512,8 @@ public class IndexUtils {
                 DoubleNumberProperty property = DoubleNumberProperty.of(a -> {
                     buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
                     buildInnerFields(a, indexParam);
+                    MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
+                    MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
                     return a;
                 });
                 properties.put(fieldName, property._toProperty());
@@ -509,6 +523,8 @@ public class IndexUtils {
                 HalfFloatNumberProperty property = HalfFloatNumberProperty.of(a -> {
                     buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
                     buildInnerFields(a, indexParam);
+                    MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
+                    MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
                     return a;
                 });
                 properties.put(fieldName, property._toProperty());
@@ -521,6 +537,8 @@ public class IndexUtils {
                     buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
                     buildInnerFields(a, indexParam);
                     a.scalingFactor(scalingFactor);
+                    MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
+                    MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
                     return a;
                 });
                 properties.put(fieldName, property._toProperty());
@@ -531,6 +549,8 @@ public class IndexUtils {
                     a.boost(indexParam.getBoost());
                     buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
                     buildInnerFields(a, indexParam);
+                    MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
+                    MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
                     return a;
                 });
                 properties.put(fieldName, property._toProperty());
@@ -538,10 +558,11 @@ public class IndexUtils {
             }
             if (FieldType.DATE.getType().equals(indexParam.getFieldType())) {
                 DateProperty property = DateProperty.of(a -> {
-                    a.boost(indexParam.getBoost());
+                    a.boost(indexParam.getBoost()).format(indexParam.getDateFormat());
                     buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
                     buildInnerFields(a, indexParam);
-                    a.format(indexParam.getDateFormat());
+                    MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
+                    MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
                     return a;
                 });
                 properties.put(fieldName, property._toProperty());
@@ -551,6 +572,7 @@ public class IndexUtils {
                 BinaryProperty property = BinaryProperty.of(a -> {
                     buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
                     buildInnerFields(a, indexParam);
+                    MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
                     return a;
                 });
                 properties.put(fieldName, property._toProperty());
@@ -558,22 +580,23 @@ public class IndexUtils {
             }
             if (FieldType.KEYWORD.getType().equals(indexParam.getFieldType())) {
                 KeywordProperty property = KeywordProperty.of(a -> {
-                    a.boost(indexParam.getBoost());
+                    a.boost(indexParam.getBoost())
+                            .normalizer(indexParam.isIgnoreCase() ? LOWERCASE_NORMALIZER : null);
                     buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
                     buildInnerFields(a, indexParam);
-                    a.normalizer(indexParam.isIgnoreCase() ? LOWERCASE_NORMALIZER : null);
+                    MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
+                    MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
                     return a;
                 });
                 properties.put(fieldName, property._toProperty());
                 return;
             }
             if (FieldType.TEXT.getType().equals(indexParam.getFieldType())) {
-                int ignoreAbove = Optional.ofNullable(indexParam.getIgnoreAbove()).orElse(DEFAULT_IGNORE_ABOVE);
                 TextProperty property = TextProperty.of(a -> {
-                    a.boost(indexParam.getBoost());
                     a.boost(indexParam.getBoost());
                     buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
                     buildInnerFields(a, indexParam);
+                    MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
                     Optional.ofNullable(indexParam.getAnalyzer()).map(String::toLowerCase).ifPresent(a::analyzer);
                     Optional.ofNullable(indexParam.getSearchAnalyzer()).map(String::toLowerCase).ifPresent(a::searchAnalyzer);
                     MyOptional.ofNullable(indexParam.getFieldData()).ifTrue(a::fielddata);
@@ -588,9 +611,11 @@ public class IndexUtils {
                     a.boost(indexParam.getBoost());
                     buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
                     buildInnerFields(a, indexParam);
+                    MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
                     a.fields(FieldType.KEYWORD.getType(), c -> c
                             .keyword(d -> {
                                 d.ignoreAbove(ignoreAbove);
+                                MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
                                 if (indexParam.isIgnoreCase()) {
                                     d.normalizer(LOWERCASE_NORMALIZER);
                                 }
@@ -610,6 +635,7 @@ public class IndexUtils {
                 WildcardProperty property = WildcardProperty.of(a -> {
                     buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
                     buildInnerFields(a, indexParam);
+                    MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
                     return a;
                 });
                 properties.put(fieldName, property._toProperty());
@@ -650,6 +676,8 @@ public class IndexUtils {
                 GeoPointProperty property = GeoPointProperty.of(a -> {
                     buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
                     buildInnerFields(a, indexParam);
+                    MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
+                    MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
                     return a;
                 });
                 properties.put(fieldName, property._toProperty());
@@ -659,6 +687,7 @@ public class IndexUtils {
                 GeoShapeProperty property = GeoShapeProperty.of(a -> {
                     buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
                     buildInnerFields(a, indexParam);
+                    MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
                     return a;
                 });
                 properties.put(fieldName, property._toProperty());
@@ -669,6 +698,8 @@ public class IndexUtils {
                     a.boost(indexParam.getBoost());
                     buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
                     buildInnerFields(a, indexParam);
+                    MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
+                    MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
                     return a;
                 });
                 properties.put(fieldName, property._toProperty());
@@ -678,6 +709,7 @@ public class IndexUtils {
                 CompletionProperty property = CompletionProperty.of(a -> {
                     buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
                     buildInnerFields(a, indexParam);
+                    MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
                     Optional.ofNullable(indexParam.getAnalyzer()).map(String::toLowerCase).ifPresent(a::analyzer);
                     Optional.ofNullable(indexParam.getSearchAnalyzer()).map(String::toLowerCase).ifPresent(a::searchAnalyzer);
                     return a;
@@ -690,6 +722,8 @@ public class IndexUtils {
                     a.boost(indexParam.getBoost());
                     buildCopyTo(a, entityInfo.isIndexEqualStage(), indexParam.getCopyToList());
                     buildInnerFields(a, indexParam);
+                    MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
+                    MyOptional.ofNullable(indexParam.getDocValues()).ifFalse(a::docValues);
                     return a;
                 });
                 properties.put(fieldName, property._toProperty());
@@ -710,6 +744,7 @@ public class IndexUtils {
                 DenseVectorProperty property = DenseVectorProperty.of(a -> {
                     Optional.ofNullable(indexParam.getDims()).ifPresent(a::dims);
                     buildInnerFields(a, indexParam);
+                    MyOptional.ofNullable(indexParam.getIndex()).ifFalse(a::index);
                     return a;
                 });
                 properties.put(fieldName, property._toProperty());
@@ -1020,6 +1055,9 @@ public class IndexUtils {
                 esIndexParam.setScalingFactor(field.getScalingFactor());
                 esIndexParam.setDims(field.getDims());
                 esIndexParam.setCopyToList(field.getCopyToList());
+                esIndexParam.setIgnoreCase(field.isIgnoreCase());
+                esIndexParam.setIndex(field.getIndex());
+                esIndexParam.setDocValues(field.getDocValues());
 
                 // 嵌套类型
                 if (FieldType.NESTED.equals(field.getFieldType()) || FieldType.OBJECT.equals(field.getFieldType())) {
@@ -1035,8 +1073,6 @@ public class IndexUtils {
                         esIndexParam.setSearchAnalyzer(field.getSearchAnalyzer());
                     }
                 }
-                esIndexParam.setIgnoreCase(field.isIgnoreCase());
-
                 // 日期处理
                 Optional.ofNullable(dateFormatMap)
                         .flatMap(i -> Optional.ofNullable(i.get(field.getColumn())))
@@ -1093,7 +1129,7 @@ public class IndexUtils {
         TypeMapping.Builder builderFromIndex = esIndexInfo.getBuilder();
         Map<String, Property> propertiesOfEntity = builderFromEntity.build().properties();
         Map<String, Property> propertiesOfIndex = builderFromIndex.build().properties();
-        return !PropertyComparator.isPropertyMapEqual(propertiesOfEntity,propertiesOfIndex);
+        return !PropertyComparator.isPropertyMapEqual(propertiesOfEntity, propertiesOfIndex);
     }
 
     /**

easy-es-springboot-test/src/main/java/org/dromara/easyes/test/entity/Document.java

@@ -156,4 +156,10 @@ public class Document extends BaseJoin {
      */
     @IndexField(fieldType = FieldType.DENSE_VECTOR, dims = 3)
     private double[] vector;
+
+    @IndexField(fieldType = FieldType.KEYWORD, index = false)
+    private String indexFalse;
+
+    @IndexField(fieldType = FieldType.KEYWORD, docValues = false)
+    private String docFalse;
 }

easy-es-springboot-test/src/test/java/org/dromara/easyes/test/other/IndexFalseTest.java

@@ -0,0 +1,70 @@
+package org.dromara.easyes.test.other;
+
+
+import co.elastic.clients.elasticsearch._types.ElasticsearchException;
+import org.dromara.easyes.common.exception.EasyEsException;
+import org.dromara.easyes.core.conditions.select.LambdaEsQueryWrapper;
+import org.dromara.easyes.test.TestEasyEsApplication;
+import org.dromara.easyes.test.entity.Document;
+import org.dromara.easyes.test.mapper.DocumentMapper;
+import org.junit.jupiter.api.*;
+import org.springframework.boot.test.context.SpringBootTest;
+
+import javax.annotation.Resource;
+
+/**
+ * index和docValues测试
+ */
+@DisplayName("index和docValues测试")
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+@SpringBootTest(classes = TestEasyEsApplication.class)
+public class IndexFalseTest {
+    @Resource
+    private DocumentMapper documentMapper;
+
+    /**
+     * index为false时，无法搜索
+     */
+    @Test
+    void testIndexFalse() {
+//        documentMapper.deleteIndex("easyes_document");
+//        documentMapper.createIndex();
+        LambdaEsQueryWrapper<Document> wrapper = new LambdaEsQueryWrapper<>();
+        wrapper.eq(Document::getIndexFalse, "777");
+        Exception exception = null;
+        try {
+            documentMapper.selectList(wrapper);
+        } catch (ElasticsearchException e) {
+            e.printStackTrace();
+            System.out.println(e.response());
+//            failed to create query: Cannot search on field [indexFalse] since it is not indexed
+            exception = e;
+        }
+        Assertions.assertNotNull(exception);
+    }
+
+    /**
+     * docValues为false时，无法聚合
+     */
+    @Test
+    void testDocValueFalse() {
+//        documentMapper.deleteIndex("easyes_document");
+//        documentMapper.createIndex();
+        Exception exception = null;
+        LambdaEsQueryWrapper<Document> wrapper = new LambdaEsQueryWrapper<>();
+        wrapper.termsAggregation(Document::getDocFalse);
+        try {
+            documentMapper.search(wrapper);
+        } catch (EasyEsException e) {
+            e.printStackTrace();
+            if (e.getCause() instanceof ElasticsearchException) {
+//                这个异常描述貌似有问题？
+//                Can't load fielddata on [docFalse] because fielddata is unsupported on fields of type [keyword]. Use doc values instead.
+                System.out.println(((ElasticsearchException) e.getCause()).response());
+            }
+            exception = e;
+        }
+        Assertions.assertNotNull(exception);
+    }
+
+}