Added file nullable and new search parameter

This commit is contained in:
xgc 2024-12-09 12:27:52 +08:00
parent 272af444ea
commit 915823fa62
17 changed files with 158 additions and 48 deletions

View File

@ -17,7 +17,7 @@
<dependency> <dependency>
<groupId>org.dromara.milvus-plus</groupId> <groupId>org.dromara.milvus-plus</groupId>
<artifactId>milvus-plus-core</artifactId> <artifactId>milvus-plus-core</artifactId>
<version>2.2.0</version> <version>2.2.2</version>
</dependency> </dependency>
</dependencies> </dependencies>
<dependencyManagement> <dependencyManagement>

View File

@ -2,13 +2,13 @@ package org.dromara.milvus.plus.mapper;
import io.milvus.v2.client.MilvusClientV2; import io.milvus.v2.client.MilvusClientV2;
import org.dromara.milvus.plus.core.mapper.BaseMilvusMapper; import org.dromara.milvus.plus.core.mapper.BaseMilvusMapper;
import org.dromara.milvus.plus.util.SpringUtils; import org.dromara.milvus.plus.util.MilvusSpringUtils;
public class MilvusMapper<T> extends BaseMilvusMapper<T> { public class MilvusMapper<T> extends BaseMilvusMapper<T> {
@Override @Override
public MilvusClientV2 getClient() { public MilvusClientV2 getClient() {
return SpringUtils.getBean(MilvusClientV2.class); return MilvusSpringUtils.getBean(MilvusClientV2.class);
} }
} }

View File

@ -27,7 +27,7 @@ public class MilvusInit extends AbstractMilvusClientBuilder implements Initializ
} }
@Override @Override
public void destroy() throws Exception { public void destroy() throws Exception {
super.close(); // super.close();
} }
public void initialize() { public void initialize() {

View File

@ -16,7 +16,7 @@ import org.dromara.milvus.plus.core.mapper.BaseMilvusMapper;
import org.dromara.milvus.plus.model.MilvusEntity; import org.dromara.milvus.plus.model.MilvusEntity;
import org.dromara.milvus.plus.model.vo.MilvusResp; import org.dromara.milvus.plus.model.vo.MilvusResp;
import org.dromara.milvus.plus.model.vo.MilvusResult; import org.dromara.milvus.plus.model.vo.MilvusResult;
import org.dromara.milvus.plus.util.SpringUtils; import org.dromara.milvus.plus.util.MilvusSpringUtils;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import java.io.Serializable; import java.io.Serializable;
@ -29,7 +29,7 @@ public class MilvusService implements IAMService,ICMService,IVecMService{
@Override @Override
public MilvusClientV2 getClient() { public MilvusClientV2 getClient() {
return SpringUtils.getBean(MilvusClientV2.class); return MilvusSpringUtils.getBean(MilvusClientV2.class);
} }
public <T> MilvusResp<List<MilvusResult<T>>> getById(Class<T> entityClass,Serializable... ids) { public <T> MilvusResp<List<MilvusResult<T>>> getById(Class<T> entityClass,Serializable... ids) {

View File

@ -11,13 +11,13 @@ import org.springframework.stereotype.Component;
*/ */
@Component @Component
public class SpringUtils implements ApplicationContextAware { public class MilvusSpringUtils implements ApplicationContextAware {
private static ApplicationContext applicationContext; private static ApplicationContext applicationContext;
@Override @Override
public void setApplicationContext(ApplicationContext applicationContext) throws BeansException { public void setApplicationContext(ApplicationContext applicationContext) throws BeansException {
SpringUtils.applicationContext = applicationContext; MilvusSpringUtils.applicationContext = applicationContext;
} }
/** /**

View File

@ -28,7 +28,7 @@
<dependency> <dependency>
<groupId>io.milvus</groupId> <groupId>io.milvus</groupId>
<artifactId>milvus-sdk-java</artifactId> <artifactId>milvus-sdk-java</artifactId>
<version>2.5.0</version> <version>2.5.1</version>
<exclusions> <exclusions>
<exclusion> <exclusion>
<groupId>org.apache.logging.log4j</groupId> <groupId>org.apache.logging.log4j</groupId>

View File

@ -42,6 +42,12 @@ public @interface MilvusField {
*/ */
boolean autoID() default false; boolean autoID() default false;
/**
* 是否允许为空
* @return
*/
boolean nullable() default false;
/** /**
* 字段描述 * 字段描述
*/ */
@ -74,6 +80,7 @@ public @interface MilvusField {
*/ */
boolean enableAnalyzer() default false; boolean enableAnalyzer() default false;
/** /**
* *
* 启用文本匹配 * 启用文本匹配

View File

@ -114,6 +114,7 @@ public class MilvusConverter {
.elementType(fieldAnnotation.elementType()) .elementType(fieldAnnotation.elementType())
.enableAnalyzer(fieldAnnotation.enableAnalyzer()) .enableAnalyzer(fieldAnnotation.enableAnalyzer())
.enableMatch(fieldAnnotation.enableMatch()) .enableMatch(fieldAnnotation.enableMatch())
.isNullable(fieldAnnotation.nullable())
.autoID(false); .autoID(false);
autoID=autoID?autoID:fieldAnnotation.autoID(); autoID=autoID?autoID:fieldAnnotation.autoID();

View File

@ -49,7 +49,7 @@ public class LambdaQueryWrapper<T> extends AbstractChainWrapper<T> implements Wr
private int roundDecimal = -1; private int roundDecimal = -1;
private long guaranteeTimestamp; private long guaranteeTimestamp;
private ConsistencyLevel consistencyLevel; private ConsistencyLevel consistencyLevel;
private boolean ignoreGrowing; private Boolean ignoreGrowing;
private MilvusClientV2 client; private MilvusClientV2 client;
private Map<String, Object> searchParams = new HashMap<>(16); private Map<String, Object> searchParams = new HashMap<>(16);
@ -57,6 +57,12 @@ public class LambdaQueryWrapper<T> extends AbstractChainWrapper<T> implements Wr
private BaseRanker ranker; private BaseRanker ranker;
private long gracefulTime;
private String groupByFieldName;
private Integer groupSize;
private Boolean strictGroupSize;
public LambdaQueryWrapper() { public LambdaQueryWrapper() {
} }
@ -767,6 +773,49 @@ public class LambdaQueryWrapper<T> extends AbstractChainWrapper<T> implements Wr
return this; return this;
} }
/**
* @param fieldName 按指定字段对搜索结果进行分组
* @return
*/
public LambdaQueryWrapper<T> groupByFieldName(String fieldName) {
this.groupByFieldName=fieldName;
return this;
}
public LambdaQueryWrapper<T> groupByFieldName(FieldFunction<T,?> fieldName) {
this.groupByFieldName=fieldName.getFieldName(fieldName);
return this;
}
// 设置保证时间戳
public LambdaQueryWrapper<T> guaranteeTimestamp(long guaranteeTimestamp) {
this.guaranteeTimestamp = guaranteeTimestamp;
return this;
}
// 设置优雅的时间毫秒
public LambdaQueryWrapper<T> gracefulTime(long gracefulTime) {
this.gracefulTime=gracefulTime;
return this;
}
// 设置是否忽略增长的段
public LambdaQueryWrapper<T> ignoreGrowing(boolean ignoreGrowing) {
this.ignoreGrowing = ignoreGrowing;
return this;
}
// 设置分组搜索中每组内返回的实体目标数量
public LambdaQueryWrapper<T> groupSize(Integer groupSize) {
this.groupSize=groupSize;
return this;
}
// 设置是否严格执行groupSize
public LambdaQueryWrapper<T> strictGroupSize(Boolean strictGroupSize) {
this.strictGroupSize=strictGroupSize;
return this;
}
/** /**
* 构建完整的搜索请求 * 构建完整的搜索请求
* @return 搜索请求对象 * @return 搜索请求对象
@ -811,6 +860,24 @@ public class LambdaQueryWrapper<T> extends AbstractChainWrapper<T> implements Wr
if (roundDecimal != -1) { if (roundDecimal != -1) {
builder.roundDecimal(roundDecimal); builder.roundDecimal(roundDecimal);
} }
if(guaranteeTimestamp>0l){
builder.guaranteeTimestamp(guaranteeTimestamp);
}
if(gracefulTime>0l){
builder.gracefulTime(gracefulTime);
}
if(ignoreGrowing!=null){
builder.ignoreGrowing(ignoreGrowing);
}
if(groupByFieldName!=null&&!groupByFieldName.isEmpty()){
builder.groupByFieldName(groupByFieldName);
}
if(groupSize!=null&&groupSize>0){
builder.groupSize(groupSize);
}
if(strictGroupSize!=null){
builder.strictGroupSize(strictGroupSize);
}
// Set other parameters as needed // Set other parameters as needed
return builder.build(); return builder.build();
} }

View File

@ -62,8 +62,7 @@ public abstract class AbstractMilvusClientBuilder implements MilvusClientBuilder
client.releaseCollection(releaseCollectionReq); client.releaseCollection(releaseCollectionReq);
} }
} }
client.close(100); client.close(5);
} }
} }

View File

@ -30,7 +30,7 @@
</developer> </developer>
</developers> </developers>
<properties> <properties>
<revision>2.2.1</revision> <revision>2.2.2</revision>
<maven.compiler.source>${java.version}</maven.compiler.source> <maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target> <maven.compiler.target>${java.version}</maven.compiler.target>
<maven-compiler.version>3.11.0</maven-compiler.version> <maven-compiler.version>3.11.0</maven-compiler.version>

View File

@ -33,7 +33,7 @@ public class MilvusInit extends AbstractMilvusClientBuilder implements Lifecycle
} }
public void stop() throws Throwable { public void stop() throws Throwable {
super.close(); // super.close();
} }
public void printBanner() { public void printBanner() {

View File

@ -28,7 +28,7 @@
<dependency> <dependency>
<groupId>org.dromara.milvus-plus</groupId> <groupId>org.dromara.milvus-plus</groupId>
<artifactId>milvus-plus-solon-plugin</artifactId> <artifactId>milvus-plus-solon-plugin</artifactId>
<version>2.1.7</version> <version>2.2.2</version>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.noear</groupId> <groupId>org.noear</groupId>

View File

@ -21,7 +21,7 @@
<dependency> <dependency>
<groupId>org.dromara.milvus-plus</groupId> <groupId>org.dromara.milvus-plus</groupId>
<artifactId>milvus-plus-boot-starter</artifactId> <artifactId>milvus-plus-boot-starter</artifactId>
<version>2.2.1</version> <version>2.2.2</version>
</dependency> </dependency>
</dependencies> </dependencies>
<dependencyManagement> <dependencyManagement>

View File

@ -19,6 +19,7 @@ import org.springframework.boot.ApplicationArguments;
import org.springframework.boot.ApplicationRunner; import org.springframework.boot.ApplicationRunner;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import java.io.IOException;
import java.util.List; import java.util.List;
import java.util.stream.Collectors; import java.util.stream.Collectors;
import java.util.stream.IntStream; import java.util.stream.IntStream;
@ -36,9 +37,11 @@ public class ApplicationRunnerTest implements ApplicationRunner {
} }
@Override @Override
public void run(ApplicationArguments args) throws InterruptedException { public void run(ApplicationArguments args) throws InterruptedException, IOException {
// insertFace();
// selectFace(12); insertFace();
// Thread.sleep(1000);
// selectFace(11);
// selectFace(11); // selectFace(11);
// delFace(11); // delFace(11);
// Thread.sleep(10000); // Thread.sleep(10000);
@ -47,31 +50,48 @@ public class ApplicationRunnerTest implements ApplicationRunner {
// vectorQuery(); // vectorQuery();
// scalarQuery(); // scalarQuery();
// update(); // update();
selectTextEmbedding(); // selectTextEmbedding();
// OkHttpClient client = new OkHttpClient();
//
// MediaType mediaType = MediaType.parse("text/plain");
// RequestBody body = RequestBody.create(mediaType, "{\"collectionName\":\"face_collection\",\"data\":[{\"person_name\":\"red\",\"person_id\":68,\"face_vector\":[0.11471413182829093,0.9506452386866311,0.3764267630711897,0.017116736332291937,0.7975288220162147,0.2837467793885893,0.45482217562314897,0.7512643321503942,0.5851774373796306,0.29498709649584076,0.5740173220271303,0.008283836490630359,0.043819261649065355,0.7510447248008454,0.26893026366305284,0.7295046143053174,0.7474013871060432,0.49231964145616947,0.6573919547221014,0.5648975702337511,0.9170353198810068,0.8043074271278763,0.8063592804980482,0.7815854305506693,0.3554478122511564,0.531480801706852,0.22560678943537482,0.5506750630287303,0.0022136957722376227,0.7276468730030954,0.6440332075744812,0.5698596907205946,0.5234591603448207,0.354993498425594,0.10620817906176727,0.07369212203402742,0.9813164411361883,0.2738860472889031,0.3271325745522793,0.050917994542803324,0.158202226640898,0.6272926953994711,0.5441414134787672,0.8910999073515409,0.8981121232379727,0.9985463611465781,0.43725436443853805,0.3120131342129935,0.24224544970671125,0.3681398853032727,0.5119269952123464,0.30916828759295356,0.6813753108853584,0.35691818319485047,0.43402512100213664,0.056672838867129594,0.7946916422488417,0.7178773219718055,0.45241404297020327,0.9306370216254345,0.09492427373318102,0.028570525877182007,0.1511031657904387,0.015414492250692469,0.06082024183250767,0.24318125198436924,0.8654445318335819,0.5892043659015123,0.019432939418877915,0.9345634565985648,0.588436900357431,0.7628378546017671,0.7127967940530804,0.5408600565378519,0.9451125627364227,0.50327646514356,0.624020091541091,0.06878519754282042,0.9023873103555076,0.06118535678364112,0.7727462265061942,0.5126902206187238,0.4101262307143809,0.5045701763198585,0.6097751212360454,0.26960758501937354,0.37397712778263625,0.16474237320190754,0.20184603623861452,0.025409236087541087,0.5683571865426373,0.47285055320039815,0.5046689191338607,0.9483211572128358,0.5880427237081676,0.7222636773561573,0.05098737933831754,0.9170210804631558,0.9631766161740742,0.9923927645980319,0.08716668133143668,0.8717887659732193,0.1638853403438041,0.8321367669259312,0.14796279048110583,0.5479872807310744,0.21622665177519096,0.4966695030300612,0.6457899721231348,0.9485180567239628,0.9643994129194424,0.31552473686825233,0.5185638828626413,0.26134044997524963,0.4307649919670753,0.27887814295025404,0.34751074846361885,0.41590517289697715,0.4460242095520486,0.6940091447850067,0.4794804720357553,0.24354302384948556,0.4511238595497853,0.40982391210503444,0.23674501807134418,0.3286329509283674,0.8590652585582985,0.9959059288831775]}]}");
// Request request = new Request.Builder()
// .url("http://154.201.90.228:19530/v2/vectordb/entities/insert")
// .post(body)
//// .addHeader("Authorization", "Bearer 6fab5641a3156d2666feba14390e4ef4b6d376b5dce91faed303eec91a4bdb82239b70b29eb252b981daa3170516245818d4ee12")
// .addHeader("Accept", "application/json")
// .addHeader("Content-Type", "application/json")
// .build();
//
// Response response = client.newCall(request).execute();
} }
private void selectTextEmbedding(){
MilvusResp<List<MilvusResult<Face>>> xx = mapper
.queryWrapper()
.textVector(Face::getText, "whats the focus of information retrieval?")
.textMatch(Face::getText,"retrieval")
.topK(2)
.query();
System.out.println("===");
}
private void selectFace(Integer temp){
MilvusResp<List<MilvusResult<Face>>> query = mapper.
queryWrapper()
.eq(Face::getTemp, temp)
.query(Face::getPersonName,Face::getTemp);
log.info("query temp 11--{}", GsonUtil.toJson(query));
LambdaQueryWrapper<Face> mapper = milvusService.ofQuery(Face.class);
MilvusResp<List<MilvusResult<Face>>> test = mapper // private void selectTextEmbedding(){
.eq(Face::getPersonName, "test") // MilvusResp<List<MilvusResult<Face>>> xx = mapper
.topK(1) // .queryWrapper()
.query(); // .textVector(Face::getText, "whats the focus of information retrieval?")
} // .textMatch(Face::getText,"retrieval")
// .topK(2)
// .query();
// System.out.println("===");
// }
// private void selectFace(Integer temp){
// MilvusResp<List<MilvusResult<Face>>> query = mapper.
// queryWrapper()
// .eq(Face::getTemp, temp)
// .query(Face::getPersonName,Face::getTemp);
// log.info("query temp 11--{}", GsonUtil.toJson(query));
//
// LambdaQueryWrapper<Face> mapper = milvusService.ofQuery(Face.class);
// MilvusResp<List<MilvusResult<Face>>> test = mapper
// .eq(Face::getSex, "")
// .topK(3)
// .query();
// log.info("query temp test--{}", GsonUtil.toJson(test));
//
// }
private void countFace(Integer temp){ private void countFace(Integer temp){
MilvusResp<Long> query = mapper. MilvusResp<Long> query = mapper.
queryWrapper() queryWrapper()
@ -84,7 +104,7 @@ public class ApplicationRunnerTest implements ApplicationRunner {
log.info("del temp 11 --{}", GsonUtil.toJson(remove)); log.info("del temp 11 --{}", GsonUtil.toJson(remove));
} }
private void insertFace() { private void insertFace() {
List<Face> faces = LongStream.range(1, 2) List<Face> faces = LongStream.range(1, 3)
.mapToObj(i -> { .mapToObj(i -> {
Face faceTmp = new Face(); Face faceTmp = new Face();
// faceTmp.setPersonId(i); // faceTmp.setPersonId(i);
@ -100,6 +120,8 @@ public class ApplicationRunnerTest implements ApplicationRunner {
faceTmp.setPerson(person); faceTmp.setPerson(person);
faceTmp.setTemp(i%2==0?11:22); faceTmp.setTemp(i%2==0?11:22);
faceTmp.setText(i % 2 == 0 ?"nformation retrieval is a field of study.":"information retrieval focuses on finding relevant information in large datasets."); faceTmp.setText(i % 2 == 0 ?"nformation retrieval is a field of study.":"information retrieval focuses on finding relevant information in large datasets.");
faceTmp.setAge(10);
faceTmp.setSex("");
return faceTmp; return faceTmp;
}) })
.collect(Collectors.toList()); .collect(Collectors.toList());
@ -109,7 +131,7 @@ public class ApplicationRunnerTest implements ApplicationRunner {
// MilvusResp<InsertResp> insert = mapper.insertWrapper() // MilvusResp<InsertResp> insert = mapper.insertWrapper()
// .partition("face_001") // .partition("face_001")
// .insert(faces.iterator()); // .insert(faces.iterator());
log.info("insert--{}", GsonUtil.toJson(insert)); // log.info("insert--{}", GsonUtil.toJson(insert));
} }
public void getByIdTest() { public void getByIdTest() {

View File

@ -9,7 +9,7 @@ import org.dromara.milvus.plus.annotation.*;
import java.util.List; import java.util.List;
@Data @Data
@MilvusCollection(name = "face_collection",level = ConsistencyLevel.STRONG) @MilvusCollection(name = "face_collection",level = ConsistencyLevel.STRONG,enableDynamicField = true)
@GenerateMilvusMapper @GenerateMilvusMapper
public class Face { public class Face {
@MilvusField( @MilvusField(
@ -60,4 +60,19 @@ public class Face {
} }
) )
private List<Float> faceVector; // 存储人脸特征的向量 private List<Float> faceVector; // 存储人脸特征的向量
//后续添加
@MilvusField(
name = "sex",
dataType = DataType.VarChar
)
private String sex;
@MilvusField(
name = "age",
dataType = DataType.Int16
)
private Integer age;
} }

View File

@ -2,9 +2,8 @@ server:
port: 8131 port: 8131
milvus: milvus:
# uri: https://in03-a5357975ab80da7.api.gcp-us-west1.zillizcloud.com # uri: https://in05-4ac29046720ae8e.serverless.gcp-us-west1.cloud.zilliz.com
uri: http://xxxx:19530 uri: http://154.201.90.228:19530
token: xxx
# token: 6fab5641a3156d2666feba14390e4ef4b6d376b5dce91faed303eec91a4bdb82239b70b29eb252b981daa3170516245818d4ee12 # token: 6fab5641a3156d2666feba14390e4ef4b6d376b5dce91faed303eec91a4bdb82239b70b29eb252b981daa3170516245818d4ee12
enable: true enable: true
packages: packages: