From 915823fa62cc12904e9233feac578b310a2b40e5 Mon Sep 17 00:00:00 2001 From: xgc Date: Mon, 9 Dec 2024 12:27:52 +0800 Subject: [PATCH] Added file nullable and new search parameter --- milvus-core-demo/pom.xml | 2 +- .../milvus/plus/mapper/MilvusMapper.java | 4 +- .../milvus/plus/service/MilvusInit.java | 2 +- .../milvus/plus/service/MilvusService.java | 4 +- ...pringUtils.java => MilvusSpringUtils.java} | 4 +- milvus-plus-core/pom.xml | 2 +- .../milvus/plus/annotation/MilvusField.java | 7 ++ .../plus/converter/MilvusConverter.java | 1 + .../core/conditions/LambdaQueryWrapper.java | 69 ++++++++++++++++- .../service/AbstractMilvusClientBuilder.java | 3 +- milvus-plus-parent/pom.xml | 2 +- .../org/dromara/solon/service/MilvusInit.java | 2 +- milvus-solon-demo/pom.xml | 2 +- milvus-spring-demo/pom.xml | 2 +- .../milvus/demo/ApplicationRunnerTest.java | 76 ++++++++++++------- .../org/dromara/milvus/demo/model/Face.java | 17 ++++- .../src/main/resources/application.yml | 7 +- 17 files changed, 158 insertions(+), 48 deletions(-) rename milvus-plus-boot-starter/src/main/java/org/dromara/milvus/plus/util/{SpringUtils.java => MilvusSpringUtils.java} (91%) diff --git a/milvus-core-demo/pom.xml b/milvus-core-demo/pom.xml index 44dc47e..94e325f 100644 --- a/milvus-core-demo/pom.xml +++ b/milvus-core-demo/pom.xml @@ -17,7 +17,7 @@ org.dromara.milvus-plus milvus-plus-core - 2.2.0 + 2.2.2 diff --git a/milvus-plus-boot-starter/src/main/java/org/dromara/milvus/plus/mapper/MilvusMapper.java b/milvus-plus-boot-starter/src/main/java/org/dromara/milvus/plus/mapper/MilvusMapper.java index 5aa67b0..05ce82f 100644 --- a/milvus-plus-boot-starter/src/main/java/org/dromara/milvus/plus/mapper/MilvusMapper.java +++ b/milvus-plus-boot-starter/src/main/java/org/dromara/milvus/plus/mapper/MilvusMapper.java @@ -2,13 +2,13 @@ package org.dromara.milvus.plus.mapper; import io.milvus.v2.client.MilvusClientV2; import org.dromara.milvus.plus.core.mapper.BaseMilvusMapper; -import org.dromara.milvus.plus.util.SpringUtils; +import org.dromara.milvus.plus.util.MilvusSpringUtils; public class MilvusMapper extends BaseMilvusMapper { @Override public MilvusClientV2 getClient() { - return SpringUtils.getBean(MilvusClientV2.class); + return MilvusSpringUtils.getBean(MilvusClientV2.class); } } diff --git a/milvus-plus-boot-starter/src/main/java/org/dromara/milvus/plus/service/MilvusInit.java b/milvus-plus-boot-starter/src/main/java/org/dromara/milvus/plus/service/MilvusInit.java index 027383a..48dc9e1 100644 --- a/milvus-plus-boot-starter/src/main/java/org/dromara/milvus/plus/service/MilvusInit.java +++ b/milvus-plus-boot-starter/src/main/java/org/dromara/milvus/plus/service/MilvusInit.java @@ -27,7 +27,7 @@ public class MilvusInit extends AbstractMilvusClientBuilder implements Initializ } @Override public void destroy() throws Exception { - super.close(); + // super.close(); } public void initialize() { diff --git a/milvus-plus-boot-starter/src/main/java/org/dromara/milvus/plus/service/MilvusService.java b/milvus-plus-boot-starter/src/main/java/org/dromara/milvus/plus/service/MilvusService.java index 9c6c4f3..d1e6c00 100644 --- a/milvus-plus-boot-starter/src/main/java/org/dromara/milvus/plus/service/MilvusService.java +++ b/milvus-plus-boot-starter/src/main/java/org/dromara/milvus/plus/service/MilvusService.java @@ -16,7 +16,7 @@ import org.dromara.milvus.plus.core.mapper.BaseMilvusMapper; import org.dromara.milvus.plus.model.MilvusEntity; import org.dromara.milvus.plus.model.vo.MilvusResp; import org.dromara.milvus.plus.model.vo.MilvusResult; -import org.dromara.milvus.plus.util.SpringUtils; +import org.dromara.milvus.plus.util.MilvusSpringUtils; import org.springframework.stereotype.Service; import java.io.Serializable; @@ -29,7 +29,7 @@ public class MilvusService implements IAMService,ICMService,IVecMService{ @Override public MilvusClientV2 getClient() { - return SpringUtils.getBean(MilvusClientV2.class); + return MilvusSpringUtils.getBean(MilvusClientV2.class); } public MilvusResp>> getById(Class entityClass,Serializable... ids) { diff --git a/milvus-plus-boot-starter/src/main/java/org/dromara/milvus/plus/util/SpringUtils.java b/milvus-plus-boot-starter/src/main/java/org/dromara/milvus/plus/util/MilvusSpringUtils.java similarity index 91% rename from milvus-plus-boot-starter/src/main/java/org/dromara/milvus/plus/util/SpringUtils.java rename to milvus-plus-boot-starter/src/main/java/org/dromara/milvus/plus/util/MilvusSpringUtils.java index 03f2fa0..7e45630 100644 --- a/milvus-plus-boot-starter/src/main/java/org/dromara/milvus/plus/util/SpringUtils.java +++ b/milvus-plus-boot-starter/src/main/java/org/dromara/milvus/plus/util/MilvusSpringUtils.java @@ -11,13 +11,13 @@ import org.springframework.stereotype.Component; */ @Component -public class SpringUtils implements ApplicationContextAware { +public class MilvusSpringUtils implements ApplicationContextAware { private static ApplicationContext applicationContext; @Override public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { - SpringUtils.applicationContext = applicationContext; + MilvusSpringUtils.applicationContext = applicationContext; } /** diff --git a/milvus-plus-core/pom.xml b/milvus-plus-core/pom.xml index 7b3eee0..f8e1b23 100644 --- a/milvus-plus-core/pom.xml +++ b/milvus-plus-core/pom.xml @@ -28,7 +28,7 @@ io.milvus milvus-sdk-java - 2.5.0 + 2.5.1 org.apache.logging.log4j diff --git a/milvus-plus-core/src/main/java/org/dromara/milvus/plus/annotation/MilvusField.java b/milvus-plus-core/src/main/java/org/dromara/milvus/plus/annotation/MilvusField.java index 1d94553..81fb3cd 100644 --- a/milvus-plus-core/src/main/java/org/dromara/milvus/plus/annotation/MilvusField.java +++ b/milvus-plus-core/src/main/java/org/dromara/milvus/plus/annotation/MilvusField.java @@ -42,6 +42,12 @@ public @interface MilvusField { */ boolean autoID() default false; + /** + * 是否允许为空 + * @return + */ + boolean nullable() default false; + /** * 字段描述 */ @@ -74,6 +80,7 @@ public @interface MilvusField { */ boolean enableAnalyzer() default false; + /** * * 启用文本匹配 diff --git a/milvus-plus-core/src/main/java/org/dromara/milvus/plus/converter/MilvusConverter.java b/milvus-plus-core/src/main/java/org/dromara/milvus/plus/converter/MilvusConverter.java index c1daf04..f6a8159 100644 --- a/milvus-plus-core/src/main/java/org/dromara/milvus/plus/converter/MilvusConverter.java +++ b/milvus-plus-core/src/main/java/org/dromara/milvus/plus/converter/MilvusConverter.java @@ -114,6 +114,7 @@ public class MilvusConverter { .elementType(fieldAnnotation.elementType()) .enableAnalyzer(fieldAnnotation.enableAnalyzer()) .enableMatch(fieldAnnotation.enableMatch()) + .isNullable(fieldAnnotation.nullable()) .autoID(false); autoID=autoID?autoID:fieldAnnotation.autoID(); diff --git a/milvus-plus-core/src/main/java/org/dromara/milvus/plus/core/conditions/LambdaQueryWrapper.java b/milvus-plus-core/src/main/java/org/dromara/milvus/plus/core/conditions/LambdaQueryWrapper.java index a6fb126..27e8ef0 100644 --- a/milvus-plus-core/src/main/java/org/dromara/milvus/plus/core/conditions/LambdaQueryWrapper.java +++ b/milvus-plus-core/src/main/java/org/dromara/milvus/plus/core/conditions/LambdaQueryWrapper.java @@ -49,7 +49,7 @@ public class LambdaQueryWrapper extends AbstractChainWrapper implements Wr private int roundDecimal = -1; private long guaranteeTimestamp; private ConsistencyLevel consistencyLevel; - private boolean ignoreGrowing; + private Boolean ignoreGrowing; private MilvusClientV2 client; private Map searchParams = new HashMap<>(16); @@ -57,6 +57,12 @@ public class LambdaQueryWrapper extends AbstractChainWrapper implements Wr private BaseRanker ranker; + private long gracefulTime; + private String groupByFieldName; + + private Integer groupSize; + private Boolean strictGroupSize; + public LambdaQueryWrapper() { } @@ -767,6 +773,49 @@ public class LambdaQueryWrapper extends AbstractChainWrapper implements Wr return this; } + /** + * @param fieldName 按指定字段对搜索结果进行分组 + * @return + */ + public LambdaQueryWrapper groupByFieldName(String fieldName) { + this.groupByFieldName=fieldName; + return this; + } + public LambdaQueryWrapper groupByFieldName(FieldFunction fieldName) { + this.groupByFieldName=fieldName.getFieldName(fieldName); + return this; + } + + // 设置保证时间戳 + public LambdaQueryWrapper guaranteeTimestamp(long guaranteeTimestamp) { + this.guaranteeTimestamp = guaranteeTimestamp; + return this; + } + + // 设置优雅的时间（毫秒） + public LambdaQueryWrapper gracefulTime(long gracefulTime) { + this.gracefulTime=gracefulTime; + return this; + } + + // 设置是否忽略增长的段 + public LambdaQueryWrapper ignoreGrowing(boolean ignoreGrowing) { + this.ignoreGrowing = ignoreGrowing; + return this; + } + + // 设置分组搜索中每组内返回的实体目标数量 + public LambdaQueryWrapper groupSize(Integer groupSize) { + this.groupSize=groupSize; + return this; + } + + // 设置是否严格执行groupSize + public LambdaQueryWrapper strictGroupSize(Boolean strictGroupSize) { + this.strictGroupSize=strictGroupSize; + return this; + } + /** * 构建完整的搜索请求 * @return 搜索请求对象 @@ -811,6 +860,24 @@ public class LambdaQueryWrapper extends AbstractChainWrapper implements Wr if (roundDecimal != -1) { builder.roundDecimal(roundDecimal); } + if(guaranteeTimestamp>0l){ + builder.guaranteeTimestamp(guaranteeTimestamp); + } + if(gracefulTime>0l){ + builder.gracefulTime(gracefulTime); + } + if(ignoreGrowing!=null){ + builder.ignoreGrowing(ignoreGrowing); + } + if(groupByFieldName!=null&&!groupByFieldName.isEmpty()){ + builder.groupByFieldName(groupByFieldName); + } + if(groupSize!=null&&groupSize>0){ + builder.groupSize(groupSize); + } + if(strictGroupSize!=null){ + builder.strictGroupSize(strictGroupSize); + } // Set other parameters as needed return builder.build(); } diff --git a/milvus-plus-core/src/main/java/org/dromara/milvus/plus/service/AbstractMilvusClientBuilder.java b/milvus-plus-core/src/main/java/org/dromara/milvus/plus/service/AbstractMilvusClientBuilder.java index 78807e3..6a76ff9 100644 --- a/milvus-plus-core/src/main/java/org/dromara/milvus/plus/service/AbstractMilvusClientBuilder.java +++ b/milvus-plus-core/src/main/java/org/dromara/milvus/plus/service/AbstractMilvusClientBuilder.java @@ -62,8 +62,7 @@ public abstract class AbstractMilvusClientBuilder implements MilvusClientBuilder client.releaseCollection(releaseCollectionReq); } } - client.close(100); - + client.close(5); } } diff --git a/milvus-plus-parent/pom.xml b/milvus-plus-parent/pom.xml index 3b10618..4b076da 100644 --- a/milvus-plus-parent/pom.xml +++ b/milvus-plus-parent/pom.xml @@ -30,7 +30,7 @@ - 2.2.1 + 2.2.2 ${java.version} ${java.version} 3.11.0 diff --git a/milvus-plus-solon-plugin/src/main/java/org/dromara/solon/service/MilvusInit.java b/milvus-plus-solon-plugin/src/main/java/org/dromara/solon/service/MilvusInit.java index 9c0417a..4b0bcab 100644 --- a/milvus-plus-solon-plugin/src/main/java/org/dromara/solon/service/MilvusInit.java +++ b/milvus-plus-solon-plugin/src/main/java/org/dromara/solon/service/MilvusInit.java @@ -33,7 +33,7 @@ public class MilvusInit extends AbstractMilvusClientBuilder implements Lifecycle } public void stop() throws Throwable { - super.close(); + // super.close(); } public void printBanner() { diff --git a/milvus-solon-demo/pom.xml b/milvus-solon-demo/pom.xml index 12c061a..0938f38 100755 --- a/milvus-solon-demo/pom.xml +++ b/milvus-solon-demo/pom.xml @@ -28,7 +28,7 @@ org.dromara.milvus-plus milvus-plus-solon-plugin - 2.1.7 + 2.2.2 org.noear diff --git a/milvus-spring-demo/pom.xml b/milvus-spring-demo/pom.xml index 1804145..e211250 100644 --- a/milvus-spring-demo/pom.xml +++ b/milvus-spring-demo/pom.xml @@ -21,7 +21,7 @@ org.dromara.milvus-plus milvus-plus-boot-starter - 2.2.1 + 2.2.2 diff --git a/milvus-spring-demo/src/main/java/org/dromara/milvus/demo/ApplicationRunnerTest.java b/milvus-spring-demo/src/main/java/org/dromara/milvus/demo/ApplicationRunnerTest.java index 427c46f..9860ccc 100644 --- a/milvus-spring-demo/src/main/java/org/dromara/milvus/demo/ApplicationRunnerTest.java +++ b/milvus-spring-demo/src/main/java/org/dromara/milvus/demo/ApplicationRunnerTest.java @@ -19,6 +19,7 @@ import org.springframework.boot.ApplicationArguments; import org.springframework.boot.ApplicationRunner; import org.springframework.stereotype.Component; +import java.io.IOException; import java.util.List; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -36,9 +37,11 @@ public class ApplicationRunnerTest implements ApplicationRunner { } @Override - public void run(ApplicationArguments args) throws InterruptedException { -// insertFace(); -// selectFace(12); + public void run(ApplicationArguments args) throws InterruptedException, IOException { + + insertFace(); +// Thread.sleep(1000); + // selectFace(11); // selectFace(11); // delFace(11); // Thread.sleep(10000); @@ -47,31 +50,48 @@ public class ApplicationRunnerTest implements ApplicationRunner { // vectorQuery(); // scalarQuery(); // update(); - selectTextEmbedding(); +// selectTextEmbedding(); +// OkHttpClient client = new OkHttpClient(); +// +// MediaType mediaType = MediaType.parse("text/plain"); +// RequestBody body = RequestBody.create(mediaType, "{\"collectionName\":\"face_collection\",\"data\":[{\"person_name\":\"red\",\"person_id\":68,\"face_vector\":[0.11471413182829093,0.9506452386866311,0.3764267630711897,0.017116736332291937,0.7975288220162147,0.2837467793885893,0.45482217562314897,0.7512643321503942,0.5851774373796306,0.29498709649584076,0.5740173220271303,0.008283836490630359,0.043819261649065355,0.7510447248008454,0.26893026366305284,0.7295046143053174,0.7474013871060432,0.49231964145616947,0.6573919547221014,0.5648975702337511,0.9170353198810068,0.8043074271278763,0.8063592804980482,0.7815854305506693,0.3554478122511564,0.531480801706852,0.22560678943537482,0.5506750630287303,0.0022136957722376227,0.7276468730030954,0.6440332075744812,0.5698596907205946,0.5234591603448207,0.354993498425594,0.10620817906176727,0.07369212203402742,0.9813164411361883,0.2738860472889031,0.3271325745522793,0.050917994542803324,0.158202226640898,0.6272926953994711,0.5441414134787672,0.8910999073515409,0.8981121232379727,0.9985463611465781,0.43725436443853805,0.3120131342129935,0.24224544970671125,0.3681398853032727,0.5119269952123464,0.30916828759295356,0.6813753108853584,0.35691818319485047,0.43402512100213664,0.056672838867129594,0.7946916422488417,0.7178773219718055,0.45241404297020327,0.9306370216254345,0.09492427373318102,0.028570525877182007,0.1511031657904387,0.015414492250692469,0.06082024183250767,0.24318125198436924,0.8654445318335819,0.5892043659015123,0.019432939418877915,0.9345634565985648,0.588436900357431,0.7628378546017671,0.7127967940530804,0.5408600565378519,0.9451125627364227,0.50327646514356,0.624020091541091,0.06878519754282042,0.9023873103555076,0.06118535678364112,0.7727462265061942,0.5126902206187238,0.4101262307143809,0.5045701763198585,0.6097751212360454,0.26960758501937354,0.37397712778263625,0.16474237320190754,0.20184603623861452,0.025409236087541087,0.5683571865426373,0.47285055320039815,0.5046689191338607,0.9483211572128358,0.5880427237081676,0.7222636773561573,0.05098737933831754,0.9170210804631558,0.9631766161740742,0.9923927645980319,0.08716668133143668,0.8717887659732193,0.1638853403438041,0.8321367669259312,0.14796279048110583,0.5479872807310744,0.21622665177519096,0.4966695030300612,0.6457899721231348,0.9485180567239628,0.9643994129194424,0.31552473686825233,0.5185638828626413,0.26134044997524963,0.4307649919670753,0.27887814295025404,0.34751074846361885,0.41590517289697715,0.4460242095520486,0.6940091447850067,0.4794804720357553,0.24354302384948556,0.4511238595497853,0.40982391210503444,0.23674501807134418,0.3286329509283674,0.8590652585582985,0.9959059288831775]}]}"); +// Request request = new Request.Builder() +// .url("http://154.201.90.228:19530/v2/vectordb/entities/insert") +// .post(body) +//// .addHeader("Authorization", "Bearer 6fab5641a3156d2666feba14390e4ef4b6d376b5dce91faed303eec91a4bdb82239b70b29eb252b981daa3170516245818d4ee12") +// .addHeader("Accept", "application/json") +// .addHeader("Content-Type", "application/json") +// .build(); +// +// Response response = client.newCall(request).execute(); } - private void selectTextEmbedding(){ - MilvusResp>> xx = mapper - .queryWrapper() - .textVector(Face::getText, "whats the focus of information retrieval?") - .textMatch(Face::getText,"retrieval") - .topK(2) - .query(); - System.out.println("==="); - } - private void selectFace(Integer temp){ - MilvusResp>> query = mapper. - queryWrapper() - .eq(Face::getTemp, temp) - .query(Face::getPersonName,Face::getTemp); - log.info("query temp 11--{}", GsonUtil.toJson(query)); - LambdaQueryWrapper mapper = milvusService.ofQuery(Face.class); - MilvusResp>> test = mapper - .eq(Face::getPersonName, "test") - .topK(1) - .query(); - } + + // private void selectTextEmbedding(){ +// MilvusResp>> xx = mapper +// .queryWrapper() +// .textVector(Face::getText, "whats the focus of information retrieval?") +// .textMatch(Face::getText,"retrieval") +// .topK(2) +// .query(); +// System.out.println("==="); +// } +// private void selectFace(Integer temp){ +// MilvusResp>> query = mapper. +// queryWrapper() +// .eq(Face::getTemp, temp) +// .query(Face::getPersonName,Face::getTemp); +// log.info("query temp 11--{}", GsonUtil.toJson(query)); +// +// LambdaQueryWrapper mapper = milvusService.ofQuery(Face.class); +// MilvusResp>> test = mapper +// .eq(Face::getSex, "男") +// .topK(3) +// .query(); +// log.info("query temp test--{}", GsonUtil.toJson(test)); +// +// } private void countFace(Integer temp){ MilvusResp query = mapper. queryWrapper() @@ -84,7 +104,7 @@ public class ApplicationRunnerTest implements ApplicationRunner { log.info("del temp 11 --{}", GsonUtil.toJson(remove)); } private void insertFace() { - List faces = LongStream.range(1, 2) + List faces = LongStream.range(1, 3) .mapToObj(i -> { Face faceTmp = new Face(); // faceTmp.setPersonId(i); @@ -100,6 +120,8 @@ public class ApplicationRunnerTest implements ApplicationRunner { faceTmp.setPerson(person); faceTmp.setTemp(i%2==0?11:22); faceTmp.setText(i % 2 == 0 ?"nformation retrieval is a field of study.":"information retrieval focuses on finding relevant information in large datasets."); + faceTmp.setAge(10); + faceTmp.setSex("男"); return faceTmp; }) .collect(Collectors.toList()); @@ -109,7 +131,7 @@ public class ApplicationRunnerTest implements ApplicationRunner { // MilvusResp insert = mapper.insertWrapper() // .partition("face_001") // .insert(faces.iterator()); - log.info("insert--{}", GsonUtil.toJson(insert)); +// log.info("insert--{}", GsonUtil.toJson(insert)); } public void getByIdTest() { diff --git a/milvus-spring-demo/src/main/java/org/dromara/milvus/demo/model/Face.java b/milvus-spring-demo/src/main/java/org/dromara/milvus/demo/model/Face.java index f9fa7ab..ec627e2 100644 --- a/milvus-spring-demo/src/main/java/org/dromara/milvus/demo/model/Face.java +++ b/milvus-spring-demo/src/main/java/org/dromara/milvus/demo/model/Face.java @@ -9,7 +9,7 @@ import org.dromara.milvus.plus.annotation.*; import java.util.List; @Data -@MilvusCollection(name = "face_collection",level = ConsistencyLevel.STRONG) +@MilvusCollection(name = "face_collection",level = ConsistencyLevel.STRONG,enableDynamicField = true) @GenerateMilvusMapper public class Face { @MilvusField( @@ -60,4 +60,19 @@ public class Face { } ) private List faceVector; // 存储人脸特征的向量 + + + //后续添加 + @MilvusField( + name = "sex", + dataType = DataType.VarChar + ) + private String sex; + + @MilvusField( + name = "age", + dataType = DataType.Int16 + ) + private Integer age; + } \ No newline at end of file diff --git a/milvus-spring-demo/src/main/resources/application.yml b/milvus-spring-demo/src/main/resources/application.yml index 9ad2c91..39e7033 100644 --- a/milvus-spring-demo/src/main/resources/application.yml +++ b/milvus-spring-demo/src/main/resources/application.yml @@ -2,9 +2,8 @@ server: port: 8131 milvus: -# uri: https://in03-a5357975ab80da7.api.gcp-us-west1.zillizcloud.com - uri: http://xxxx:19530 - token: xxx +# uri: https://in05-4ac29046720ae8e.serverless.gcp-us-west1.cloud.zilliz.com + uri: http://154.201.90.228:19530 # token: 6fab5641a3156d2666feba14390e4ef4b6d376b5dce91faed303eec91a4bdb82239b70b29eb252b981daa3170516245818d4ee12 enable: true packages: @@ -12,4 +11,4 @@ milvus: open-log: true # log-level: WARN # username: db_a5357975ab80da7 -# password: mima@123 \ No newline at end of file +# password: mima@123