重构,提取出core模块,增加core-demo,以便用户更灵活扩展

This commit is contained in:
xgc 2024-05-14 21:02:12 +08:00
parent a6590d485b
commit 01158bd39a
44 changed files with 502 additions and 104 deletions

67
milvus-core-demo/pom.xml Normal file
View File

@ -0,0 +1,67 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>io.github.javpower</groupId>
<artifactId>milvus-core-demo</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>milvus-core-demo</name>
<description>milvus-core-demo</description>
<properties>
<java.version>1.8</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<spring-boot.version>2.6.13</spring-boot.version>
</properties>
<dependencies>
<dependency>
<groupId>io.github.javpower</groupId>
<artifactId>milvus-plus-core</artifactId>
<version>2.4.0-SNAPSHOT</version>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-dependencies</artifactId>
<version>${spring-boot.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${spring-boot.version}</version>
<configuration>
<mainClass>io.github.javpower.milvus.demo.MilvusDemoApplication</mainClass>
<skip>true</skip>
</configuration>
<executions>
<execution>
<id>repackage</id>
<goals>
<goal>repackage</goal>
</goals>
</execution>
</executions>
</plugin>
</plugins>
</build>
</project>

View File

@ -0,0 +1,39 @@
package io.github.javpower.milvus.demo.java;
import com.alibaba.fastjson.JSONObject;
import com.google.common.collect.Lists;
import io.github.javpower.milvus.demo.model.Face;
import io.github.javpower.milvus.plus.core.mapper.BaseMilvusMapper;
import io.github.javpower.milvus.plus.model.MilvusProperties;
import io.github.javpower.milvus.plus.model.vo.MilvusResp;
import io.github.javpower.milvus.plus.model.vo.MilvusResult;
import io.github.javpower.milvus.plus.service.impl.MilvusClientBuild;
import io.milvus.v2.client.MilvusClientV2;
import java.util.List;
public class JavaTest {
public static void main(String[] args) throws InterruptedException {
MilvusProperties properties=new MilvusProperties();
properties.setEnable(true);
properties.setUri("xxxxx");
properties.setToken("xxxx");
properties.setPackages(Lists.newArrayList("io.github.javpower.milvus.demo.model"));
MilvusClientBuild build = new MilvusClientBuild(properties);
build.initialize();
MilvusClientV2 client = build.getClient();
BaseMilvusMapper<Face> mapper=new BaseMilvusMapper<Face>() {
@Override
public MilvusClientV2 getClient() {
return client;
}
};
//标量查询
MilvusResp<List<MilvusResult<Face>>> query2 = mapper.queryWrapper()
.eq(Face::getPersonId, 2L)
.topK(3)
.query();
System.out.println("标量查询 query--queryWrapper---{}"+JSONObject.toJSONString(query2));
build.close();
}
}

View File

@ -45,41 +45,9 @@
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>3.24.0</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.22</version>
</dependency>
<dependency>
<groupId>io.milvus</groupId>
<artifactId>milvus-sdk-java</artifactId>
<version>2.4.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-csv</artifactId>
</exclusion>
<exclusion>
<groupId>com.azure</groupId>
<artifactId>azure-storage-blob</artifactId>
</exclusion>
<exclusion>
<groupId>com.azure</groupId>
<artifactId>azure-identity</artifactId>
</exclusion>
</exclusions>
<groupId>io.github.javpower</groupId>
<artifactId>milvus-plus-core</artifactId>
<version>2.4.0-SNAPSHOT</version>
</dependency>
<dependency>
<groupId>net.dreamlu</groupId>

View File

@ -1,32 +0,0 @@
package io.github.javpower.milvus.plus.config;
import io.milvus.v2.client.ConnectConfig;
import io.milvus.v2.client.MilvusClientV2;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* @author xgc
**/
@Configuration
public class MilvusConfig {
private final MilvusProperties properties;
public MilvusConfig(MilvusProperties properties) {
this.properties = properties;
}
@Bean
public MilvusClientV2 milvusClientV2() {
if(!properties.isEnable()){
return null;
}
ConnectConfig connectConfig = ConnectConfig.builder()
.uri(properties.getUri())
.token(properties.getToken())
.build();
return new MilvusClientV2(connectConfig);
}
}

View File

@ -1,4 +1,4 @@
package io.github.javpower.milvus.plus.config;
package io.github.javpower.milvus.plus.entity;
import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;

View File

@ -0,0 +1,15 @@
package io.github.javpower.milvus.plus.mapper;
import io.github.javpower.milvus.plus.core.mapper.BaseMilvusMapper;
import io.github.javpower.milvus.plus.util.SpringUtils;
import io.milvus.v2.client.MilvusClientV2;
import org.springframework.stereotype.Component;
@Component
public class MilvusMapper<T> extends BaseMilvusMapper<T> {
@Override
public MilvusClientV2 getClient() {
return SpringUtils.getBean(MilvusClientV2.class);
}
}

View File

@ -0,0 +1,41 @@
package io.github.javpower.milvus.plus.service;
import io.github.javpower.milvus.plus.entity.MilvusProperties;
import io.milvus.v2.client.MilvusClientV2;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Service;
import javax.annotation.PostConstruct;
import javax.annotation.PreDestroy;
@Service
public class MilvusInit extends AbstractMilvusClientBuilder {
@Autowired
private MilvusProperties milvusProperties;
private MilvusClientV2 client;
// Spring会调用这个方法来初始化client
@PostConstruct
public void initialize() {
io.github.javpower.milvus.plus.model.MilvusProperties milvusProperties1=new io.github.javpower.milvus.plus.model.MilvusProperties();
BeanUtils.copyProperties(milvusProperties,milvusProperties1);
super.setProperties(milvusProperties1);
super.initialize();
client = getClient();
}
// Spring会调用这个方法来关闭client
@PreDestroy
public void close() throws InterruptedException {
super.close();
}
@Bean
public MilvusClientV2 milvusClientV2() {
return client;
}
}

197
milvus-plus-core/pom.xml Normal file
View File

@ -0,0 +1,197 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>io.github.javpower</groupId>
<artifactId>milvus-plus-core</artifactId>
<version>2.4.0-SNAPSHOT</version>
<name>milvus-plus-core</name>
<description>a tool about milvus-plus</description>
<url>https://github.com/javpower/milvus-plus</url>
<licenses>
<license>
<name> The Apache Software License, Version 2.0 </name>
<url> http://www.apache.org/licenses/LICENSE-2.0.txt </url>
<distribution> repo </distribution>
</license>
</licenses>
<scm>
<url>https://github.com/javpower/milvus-plus-boot-starter</url>
<connection>scm:git@github.com/javpower/milvus-plus.git</connection>
<developerConnection>scm:git@github.com/javpower/milvus-plus.git</developerConnection>
</scm>
<developers>
<developer>
<name>gc.x</name>
<email>javpower@163.com</email>
<organization> https://github.com/javpower</organization>
<timezone>+8</timezone>
</developer>
</developers>
<properties>
<java.version>1.8</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<spring-boot.version>2.7.13</spring-boot.version>
<mica-auto.vaersion>2.3.2</mica-auto.vaersion>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId>
</dependency>
<dependency>
<groupId>com.google.protobuf</groupId>
<artifactId>protobuf-java</artifactId>
<version>3.24.0</version>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<version>1.18.22</version>
</dependency>
<dependency>
<groupId>io.milvus</groupId>
<artifactId>milvus-sdk-java</artifactId>
<version>2.4.0</version>
<exclusions>
<exclusion>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-slf4j-impl</artifactId>
</exclusion>
<exclusion>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
</exclusion>
<exclusion>
<groupId>com.fasterxml.jackson.dataformat</groupId>
<artifactId>jackson-dataformat-csv</artifactId>
</exclusion>
<exclusion>
<groupId>com.azure</groupId>
<artifactId>azure-storage-blob</artifactId>
</exclusion>
<exclusion>
<groupId>com.azure</groupId>
<artifactId>azure-identity</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>net.dreamlu</groupId>
<artifactId>mica-auto</artifactId>
<version>${mica-auto.vaersion}</version>
<scope>provided</scope>
</dependency>
</dependencies>
<dependencyManagement>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-dependencies</artifactId>
<version>${spring-boot.version}</version>
<type>pom</type>
<scope>import</scope>
</dependency>
</dependencies>
</dependencyManagement>
<distributionManagement>
<snapshotRepository>
<id>ossrh</id>
<url>https://s01.oss.sonatype.org/content/repositories/snapshots</url>
</snapshotRepository>
<repository>
<id>ossrh</id>
<url> https://s01.oss.sonatype.org/service/local/staging/deploy/maven2/</url>
</repository>
</distributionManagement>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.8.1</version>
<configuration>
<source>1.8</source>
<target>1.8</target>
<encoding>UTF-8</encoding>
</configuration>
</plugin>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
<version>${spring-boot.version}</version>
</plugin>
<plugin>
<groupId>org.sonatype.plugins</groupId>
<artifactId>nexus-staging-maven-plugin</artifactId>
<version>1.6.7</version>
<extensions>true</extensions>
<configuration>
<serverId>ossrh</serverId>
<nexusUrl>https://s01.oss.sonatype.org/</nexusUrl>
<autoReleaseAfterClose>true</autoReleaseAfterClose>
</configuration>
</plugin>
<plugin>
<groupId> org.apache.maven.plugins</groupId>
<artifactId>maven-source-plugin</artifactId>
<version>2.2.1</version>
<executions>
<execution>
<id>attach-sources</id>
<goals>
<goal>jar-no-fork</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-javadoc-plugin</artifactId>
<version>2.9.1</version>
<configuration>
<show>private</show>
<nohelp>true</nohelp>
<charset>UTF-8</charset>
<encoding>UTF-8</encoding>
<docencoding>UTF-8</docencoding>
<additionalparam>-Xdoclint:none</additionalparam>
</configuration>
<executions>
<execution>
<phase>package</phase>
<goals>
<goal>jar</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-gpg-plugin</artifactId>
<version>1.5</version>
<executions>
<execution>
<id>sign-artifacts</id>
<phase>verify</phase>
<goals>
<goal>sign</goal>
</goals>
</execution>
</executions>
</plugin>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-deploy-plugin</artifactId>
<version>2.8.2</version>
</plugin>
</plugins>
</build>
</project>

View File

@ -6,7 +6,6 @@ import io.github.javpower.milvus.plus.cache.MilvusCache;
import io.github.javpower.milvus.plus.core.conditions.*;
import io.github.javpower.milvus.plus.model.vo.MilvusResp;
import io.github.javpower.milvus.plus.model.vo.MilvusResult;
import io.github.javpower.milvus.plus.util.SpringUtils;
import io.milvus.v2.client.MilvusClientV2;
import io.milvus.v2.service.vector.response.DeleteResp;
import io.milvus.v2.service.vector.response.InsertResp;
@ -22,7 +21,11 @@ import java.util.List;
* @author xgc
**/
@Slf4j
public class MilvusMapper<T> {
public abstract class BaseMilvusMapper<T> {
public abstract MilvusClientV2 getClient();
/**
* 创建搜索构建器实例
@ -91,10 +94,9 @@ public class MilvusMapper<T> {
}
ConversionCache conversionCache = MilvusCache.milvusCache.get(entityType.getName());
String collectionName = conversionCache == null ? null : conversionCache.getCollectionName();
// 使用SpringUtil获取MilvusClient实例
MilvusClientV2 client = SpringUtils.getBean(MilvusClientV2.class);
// 初始化构建器实例
wrapper.init(collectionName, client, conversionCache, entityType);
MilvusClientV2 client = getClient();
wrapper.init(collectionName,client, conversionCache, entityType);
return wrapper.wrapper();
}

View File

@ -0,0 +1,16 @@
package io.github.javpower.milvus.plus.model;
import lombok.Data;
import java.util.List;
/**
* @author xgc
**/
@Data
public class MilvusProperties {
private boolean enable;
private String uri;
private String token;
private List<String> packages;
}

View File

@ -1,12 +1,13 @@
package io.github.javpower.milvus.plus.config;
package io.github.javpower.milvus.plus.service;
import io.github.javpower.milvus.plus.annotation.MilvusCollection;
import io.github.javpower.milvus.plus.builder.CollectionSchemaBuilder;
import io.github.javpower.milvus.plus.cache.CollectionToPrimaryCache;
import io.github.javpower.milvus.plus.converter.MilvusEntityConverter;
import io.github.javpower.milvus.plus.model.MilvusEntity;
import io.github.javpower.milvus.plus.model.MilvusProperties;
import io.milvus.exception.MilvusException;
import io.milvus.v2.client.ConnectConfig;
import io.milvus.v2.client.MilvusClientV2;
import io.milvus.v2.common.IndexParam;
import io.milvus.v2.service.collection.request.*;
@ -18,7 +19,6 @@ import org.springframework.core.io.support.ResourcePatternResolver;
import org.springframework.core.type.classreading.CachingMetadataReaderFactory;
import org.springframework.core.type.classreading.MetadataReader;
import org.springframework.core.type.classreading.MetadataReaderFactory;
import org.springframework.stereotype.Service;
import org.springframework.util.ClassUtils;
import java.util.ArrayList;
@ -27,18 +27,52 @@ import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
/**
* @author xgc
**/
@Service
@Slf4j
public class MilvusInit implements AutoCloseable {
public abstract class AbstractMilvusClientBuilder implements MilvusClientBuilder {
protected MilvusProperties properties;
protected MilvusClientV2 client;
private final static String CLASS="*.class";
public final MilvusClientV2 client;
public MilvusInit(MilvusClientV2 client, MilvusProperties properties) {
this.client = client;
public void setProperties(MilvusProperties properties) {
this.properties = properties;
}
@Override
public void initialize() {
if (properties.isEnable()) {
ConnectConfig connectConfig = ConnectConfig.builder()
.uri(properties.getUri())
.token(properties.getToken())
.build();
client = new MilvusClientV2(connectConfig);
// 初始化逻辑
handler();
}
}
@Override
public void close() throws InterruptedException {
if (client != null) {
//释放集合+释放client
Set<String> co = CollectionToPrimaryCache.collectionToPrimary.keySet();
if(co.size()>0){
for (String name : co) {
ReleaseCollectionReq releaseCollectionReq = ReleaseCollectionReq.builder()
.collectionName(name)
.build();
client.releaseCollection(releaseCollectionReq);
}
}
client.close(100);
}
}
public void handler(){
if(client!=null){
List<Class<?>> classes = getClass(properties.getPackages());
performBusinessLogic(classes);
@ -46,19 +80,10 @@ public class MilvusInit implements AutoCloseable {
}
@Override
public void close() throws InterruptedException {
//释放集合+释放client
Set<String> co = CollectionToPrimaryCache.collectionToPrimary.keySet();
if(co.size()>0){
for (String name : co) {
ReleaseCollectionReq releaseCollectionReq = ReleaseCollectionReq.builder()
.collectionName(name)
.build();
client.releaseCollection(releaseCollectionReq);
}
}
client.close(10);
public MilvusClientV2 getClient() {
return client;
}
//获取指定包下实体类
private List<Class<?>> getClass(List<String> packages){
List<Class<?>> res=new ArrayList<>();
@ -152,5 +177,4 @@ public class MilvusInit implements AutoCloseable {
}
}
}

View File

@ -0,0 +1,9 @@
package io.github.javpower.milvus.plus.service;
import io.milvus.v2.client.MilvusClientV2;
public interface MilvusClientBuilder {
void initialize();
void close() throws InterruptedException;
MilvusClientV2 getClient();
}

View File

@ -0,0 +1,12 @@
package io.github.javpower.milvus.plus.service.impl;
import io.github.javpower.milvus.plus.model.MilvusProperties;
import io.github.javpower.milvus.plus.service.AbstractMilvusClientBuilder;
public class MilvusClientBuild extends AbstractMilvusClientBuilder {
public MilvusClientBuild(MilvusProperties properties) {
super.setProperties(properties);
}
}

View File

@ -3,10 +3,10 @@
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<groupId>io.github.javpower</groupId>
<artifactId>milvus-demo</artifactId>
<artifactId>milvus-spring-demo</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>milvus-demo</name>
<description>milvus-demo</description>
<name>milvus-spring-demo</name>
<description>milvus-spring-demo</description>
<properties>
<java.version>1.8</java.version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>

View File

@ -0,0 +1,38 @@
package io.github.javpower.milvus.demo.model;
import io.github.javpower.milvus.plus.annotation.ExtraParam;
import io.github.javpower.milvus.plus.annotation.MilvusCollection;
import io.github.javpower.milvus.plus.annotation.MilvusField;
import io.github.javpower.milvus.plus.annotation.MilvusIndex;
import io.milvus.v2.common.DataType;
import io.milvus.v2.common.IndexParam;
import lombok.Data;
import java.util.List;
@Data
@MilvusCollection(name = "face_collection") // 指定Milvus集合的名称
public class Face {
@MilvusField(
name = "person_id", // 字段名称
dataType = DataType.Int64, // 数据类型为64位整数
isPrimaryKey = true, // 标记为主键
autoID = false // 假设这个ID是自动生成的
)
private Long personId; // 人员的唯一标识符
@MilvusField(
name = "face_vector", // 字段名称
dataType = DataType.FloatVector, // 数据类型为浮点型向量
dimension = 128 // 向量维度假设人脸特征向量的维度是128
)
@MilvusIndex(
indexType = IndexParam.IndexType.IVF_FLAT, // 使用IVF_FLAT索引类型
metricType = IndexParam.MetricType.L2, // 使用L2距离度量类型
indexName = "face_index", // 索引名称
extraParams = { // 指定额外的索引参数
@ExtraParam(key = "nlist", value = "100") // 例如IVF的nlist参数
}
)
private List<Float> faceVector; // 存储人脸特征的向量
}

View File

@ -1,7 +1,7 @@
package io.github.javpower.milvus.demo.test;
import io.github.javpower.milvus.demo.model.Face;
import io.github.javpower.milvus.plus.core.mapper.MilvusMapper;
import io.github.javpower.milvus.plus.mapper.MilvusMapper;
import org.springframework.stereotype.Component;
@Component

View File

@ -2,8 +2,8 @@ server:
port: 8131
milvus:
uri: http://xxxxx:19530
token: root:xxxx
uri: https://in03-a5357975ab80da7.api.gcp-us-west1.zillizcloud.com
token: XXXx
enable: true
packages:
- io.github.javpower.milvus.demo.model

View File

@ -10,8 +10,10 @@
<packaging>pom</packaging>
<modules>
<module>milvus-demo</module>
<module>milvus-core-demo</module>
<module>milvus-spring-demo</module>
<module>milvus-plus-boot-starter</module>
<module>milvus-plus-core</module>
</modules>
</project>