mirror of
https://gitee.com/milvus-io/milvus.git
synced 2025-12-08 01:58:34 +08:00
test: [GoSDK][2.5] Sync gosdk test commits from master branch (#39838)
Cherry-pick from master pr: #39484 #39570 #39537 #39824 #39579 --------- Signed-off-by: ThreadDao <yufen.zong@zilliz.com> Signed-off-by: zhuwenxing <wenxing.zhu@zilliz.com> Signed-off-by: zhuwenxing <wxzhuyeah@gmail.com> Co-authored-by: zhuwenxing <wenxing.zhu@zilliz.com>
This commit is contained in:
parent
7ea0af572a
commit
7fb69960d1
@ -87,7 +87,7 @@ func (c *Client) DescribeDatabase(ctx context.Context, option DescribeDatabaseOp
|
||||
return db, err
|
||||
}
|
||||
|
||||
func (c *Client) AlterDatabaseProperies(ctx context.Context, option AlterDatabasePropertiesOption, callOptions ...grpc.CallOption) error {
|
||||
func (c *Client) AlterDatabaseProperties(ctx context.Context, option AlterDatabasePropertiesOption, callOptions ...grpc.CallOption) error {
|
||||
req := option.Request()
|
||||
|
||||
return c.callService(func(milvusService milvuspb.MilvusServiceClient) error {
|
||||
|
||||
@ -153,7 +153,7 @@ func (s *DatabaseSuite) TestAlterDatabaseProperties() {
|
||||
return merr.Success(), nil
|
||||
}).Once()
|
||||
|
||||
err := s.client.AlterDatabaseProperies(ctx, NewAlterDatabasePropertiesOption(dbName).WithProperty(key, value))
|
||||
err := s.client.AlterDatabaseProperties(ctx, NewAlterDatabasePropertiesOption(dbName).WithProperty(key, value))
|
||||
s.NoError(err)
|
||||
})
|
||||
|
||||
@ -161,7 +161,7 @@ func (s *DatabaseSuite) TestAlterDatabaseProperties() {
|
||||
dbName := fmt.Sprintf("dt_%s", s.randString(6))
|
||||
s.mock.EXPECT().AlterDatabase(mock.Anything, mock.Anything).Return(nil, merr.WrapErrServiceInternal("mocked")).Once()
|
||||
|
||||
err := s.client.AlterDatabaseProperies(ctx, NewAlterDatabasePropertiesOption(dbName).WithProperty("key", "value"))
|
||||
err := s.client.AlterDatabaseProperties(ctx, NewAlterDatabasePropertiesOption(dbName).WithProperty("key", "value"))
|
||||
s.Error(err)
|
||||
})
|
||||
}
|
||||
|
||||
@ -1 +1,191 @@
|
||||
## go_client
|
||||
# Milvus Go Client Test Framework
|
||||
|
||||
## Overview
|
||||
This is a comprehensive test framework for the Milvus Go Client, designed to validate various functionalities of the Milvus vector database client. The framework provides a structured approach to writing tests with reusable components and helper functions.
|
||||
|
||||
## Framework Architecture
|
||||
|
||||
### Directory Structure
|
||||
```
|
||||
/go_client/
|
||||
├── testcases/ # Main test cases
|
||||
│ ├── helper/ # Helper functions and utilities
|
||||
│ │ ├── helper.go
|
||||
│ │ ├── data_helper.go
|
||||
│ │ └── collection_helper.go
|
||||
│ ├── search_test.go # Search functionality tests
|
||||
│ ├── index_test.go # Index management tests
|
||||
│ └── ...
|
||||
├── common/ # Common utilities and constants
|
||||
└── base/ # Base infrastructure code
|
||||
```
|
||||
|
||||
### Key Components
|
||||
- **Collection Preparation**: Utilities for creating and managing collections
|
||||
- **Data Generation**: Tools for generating test data
|
||||
- **Helper Functions**: Common operations and validations
|
||||
- **Test Cases**: Organized by functionality
|
||||
|
||||
## Writing Test Cases
|
||||
|
||||
### Basic Test Structure
|
||||
```go
|
||||
func TestYourFeature(t *testing.T) {
|
||||
// 1. Setup context and client
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
// 2. Prepare collection
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(
|
||||
ctx, t, mc,
|
||||
hp.NewCreateCollectionParams(hp.Int64Vec),
|
||||
hp.TNewFieldsOption(),
|
||||
hp.TNewSchemaOption(),
|
||||
)
|
||||
|
||||
// 3. Insert test data
|
||||
prepare.InsertData(ctx, t, mc,
|
||||
hp.NewInsertParams(schema),
|
||||
hp.TNewDataOption(),
|
||||
)
|
||||
|
||||
// 4. Execute test operations
|
||||
// ... your test logic here ...
|
||||
|
||||
// 5. Validate results
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, expected, actual)
|
||||
}
|
||||
```
|
||||
|
||||
### Using Custom Parameters
|
||||
|
||||
1. **Collection Creation Parameters**
|
||||
```go
|
||||
fieldsOption := hp.TNewFieldsOption().
|
||||
TWithEnableAnalyzer(true).
|
||||
TWithAnalyzerParams(map[string]any{
|
||||
"tokenizer": "standard",
|
||||
})
|
||||
|
||||
schemaOption := hp.TNewSchemaOption().
|
||||
TWithEnableDynamicField(true).
|
||||
TWithDescription("Custom schema").
|
||||
TWithAutoID(false)
|
||||
```
|
||||
|
||||
2. **Data Insertion Options**
|
||||
```go
|
||||
insertOption := hp.TNewDataOption().
|
||||
TWithNb(1000). // Number of records
|
||||
TWithDim(128). // Vector dimension
|
||||
TWithStart(100). // Starting ID
|
||||
TWithMaxLen(256). // Maximum length
|
||||
TWithTextLang("en") // Text language
|
||||
```
|
||||
|
||||
3. **Index Parameters**
|
||||
```go
|
||||
indexParams := hp.TNewIndexParams(schema).
|
||||
TWithFieldIndex(map[string]index.Index{
|
||||
common.DefaultVectorFieldName: index.NewIVFSQIndex(
|
||||
&index.IVFSQConfig{
|
||||
MetricType: entity.L2,
|
||||
NList: 128,
|
||||
},
|
||||
),
|
||||
})
|
||||
```
|
||||
|
||||
4. **Search Parameters**
|
||||
```go
|
||||
searchOpt := client.NewSearchOption(schema.CollectionName, 100, vectors).
|
||||
WithOffset(0).
|
||||
WithLimit(100).
|
||||
WithConsistencyLevel(entity.ClStrong).
|
||||
WithFilter("int64 >= 100").
|
||||
WithOutputFields([]string{"*"}).
|
||||
WithSearchParams(map[string]any{
|
||||
"nprobe": 16,
|
||||
"ef": 64,
|
||||
})
|
||||
```
|
||||
|
||||
## Adding New Parameters
|
||||
|
||||
1. **Define New Option Type**
|
||||
```go
|
||||
// In helper/data_helper.go
|
||||
type YourNewOption struct {
|
||||
newParam1 string
|
||||
newParam2 int
|
||||
}
|
||||
```
|
||||
|
||||
2. **Add Constructor**
|
||||
```go
|
||||
func TNewYourOption() *YourNewOption {
|
||||
return &YourNewOption{
|
||||
newParam1: "default",
|
||||
newParam2: 0,
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
3. **Add Parameter Methods**
|
||||
```go
|
||||
func (opt *YourNewOption) TWithNewParam1(value string) *YourNewOption {
|
||||
opt.newParam1 = value
|
||||
return opt
|
||||
}
|
||||
|
||||
func (opt *YourNewOption) TWithNewParam2(value int) *YourNewOption {
|
||||
opt.newParam2 = value
|
||||
return opt
|
||||
}
|
||||
```
|
||||
|
||||
## Best Practices
|
||||
|
||||
1. **Test Organization**
|
||||
- Group related tests in the same file
|
||||
- Use clear and descriptive test names
|
||||
- Add comments explaining test purpose
|
||||
|
||||
2. **Data Generation**
|
||||
- Use helper functions for generating test data
|
||||
- Ensure data is appropriate for the test case
|
||||
- Clean up test data after use
|
||||
|
||||
3. **Error Handling**
|
||||
- Use `common.CheckErr` for consistent error checking
|
||||
- Test both success and failure scenarios
|
||||
- Validate error messages when appropriate
|
||||
|
||||
4. **Performance Considerations**
|
||||
- Use appropriate timeouts
|
||||
- Clean up resources after tests
|
||||
- Consider test execution time
|
||||
|
||||
## Running Tests
|
||||
|
||||
```bash
|
||||
# Run all tests
|
||||
go test ./testcases/...
|
||||
|
||||
# Run specific test
|
||||
go test -run TestYourFeature ./testcases/
|
||||
|
||||
# Run with verbose output
|
||||
go test -v ./testcases/...
|
||||
```
|
||||
|
||||
## Contributing
|
||||
1. Follow the existing code structure
|
||||
2. Add comprehensive test cases
|
||||
3. Document new parameters and options
|
||||
4. Update this README for significant changes
|
||||
5. Ensure code quality standards:
|
||||
- Run `golangci-lint run` to check for style mistakes
|
||||
- Use `gofmt -w your/code/path` to format your code before submitting
|
||||
- CI will verify both golint and go format compliance
|
||||
@ -88,14 +88,14 @@ func (mc *MilvusClient) Close(ctx context.Context) error {
|
||||
|
||||
// -- database --
|
||||
|
||||
// UsingDatabase list all database in milvus cluster.
|
||||
func (mc *MilvusClient) UsingDatabase(ctx context.Context, option client.UseDatabaseOption) error {
|
||||
// UseDatabase list all database in milvus cluster.
|
||||
func (mc *MilvusClient) UseDatabase(ctx context.Context, option client.UseDatabaseOption) error {
|
||||
err := mc.mClient.UseDatabase(ctx, option)
|
||||
return err
|
||||
}
|
||||
|
||||
// ListDatabases list all database in milvus cluster.
|
||||
func (mc *MilvusClient) ListDatabases(ctx context.Context, option client.ListDatabaseOption, callOptions ...grpc.CallOption) ([]string, error) {
|
||||
// ListDatabase list all database in milvus cluster.
|
||||
func (mc *MilvusClient) ListDatabase(ctx context.Context, option client.ListDatabaseOption, callOptions ...grpc.CallOption) ([]string, error) {
|
||||
databaseNames, err := mc.mClient.ListDatabase(ctx, option, callOptions...)
|
||||
return databaseNames, err
|
||||
}
|
||||
@ -112,6 +112,24 @@ func (mc *MilvusClient) DropDatabase(ctx context.Context, option client.DropData
|
||||
return err
|
||||
}
|
||||
|
||||
// DescribeDatabase describe database with the given db name.
|
||||
func (mc *MilvusClient) DescribeDatabase(ctx context.Context, option client.DescribeDatabaseOption, callOptions ...grpc.CallOption) (*entity.Database, error) {
|
||||
database, err := mc.mClient.DescribeDatabase(ctx, option, callOptions...)
|
||||
return database, err
|
||||
}
|
||||
|
||||
// AlterDatabaseProperties alter database properties
|
||||
func (mc *MilvusClient) AlterDatabaseProperties(ctx context.Context, option client.AlterDatabasePropertiesOption, callOptions ...grpc.CallOption) error {
|
||||
err := mc.mClient.AlterDatabaseProperties(ctx, option, callOptions...)
|
||||
return err
|
||||
}
|
||||
|
||||
// DropDatabaseProperties drop database properties
|
||||
func (mc *MilvusClient) DropDatabaseProperties(ctx context.Context, option client.DropDatabasePropertiesOption, callOptions ...grpc.CallOption) error {
|
||||
err := mc.mClient.AlterDatabaseProperties(ctx, option, callOptions...)
|
||||
return err
|
||||
}
|
||||
|
||||
// -- collection --
|
||||
|
||||
// CreateCollection Create Collection
|
||||
@ -144,6 +162,36 @@ func (mc *MilvusClient) DropCollection(ctx context.Context, option client.DropCo
|
||||
return err
|
||||
}
|
||||
|
||||
// RenameCollection Rename Collection
|
||||
func (mc *MilvusClient) RenameCollection(ctx context.Context, option client.RenameCollectionOption, callOptions ...grpc.CallOption) error {
|
||||
err := mc.mClient.RenameCollection(ctx, option, callOptions...)
|
||||
return err
|
||||
}
|
||||
|
||||
// AlterCollectionProperties Alter collection properties
|
||||
func (mc *MilvusClient) AlterCollectionProperties(ctx context.Context, option client.AlterCollectionPropertiesOption, callOptions ...grpc.CallOption) error {
|
||||
err := mc.mClient.AlterCollectionProperties(ctx, option, callOptions...)
|
||||
return err
|
||||
}
|
||||
|
||||
// DropCollectionProperties Drop collection properties
|
||||
func (mc *MilvusClient) DropCollectionProperties(ctx context.Context, option client.DropCollectionPropertiesOption, callOptions ...grpc.CallOption) error {
|
||||
err := mc.mClient.DropCollectionProperties(ctx, option, callOptions...)
|
||||
return err
|
||||
}
|
||||
|
||||
// AlterCollectionField Alter collection field
|
||||
func (mc *MilvusClient) AlterCollectionField(ctx context.Context, option client.AlterCollectionFieldPropertiesOption, callOptions ...grpc.CallOption) error {
|
||||
err := mc.mClient.AlterCollectionFieldProperty(ctx, option, callOptions...)
|
||||
return err
|
||||
}
|
||||
|
||||
// GetCollectionStats Get collection stats
|
||||
func (mc *MilvusClient) GetCollectionStats(ctx context.Context, option client.GetCollectionOption) (map[string]string, error) {
|
||||
stats, err := mc.mClient.GetCollectionStats(ctx, option)
|
||||
return stats, err
|
||||
}
|
||||
|
||||
// -- partition --
|
||||
|
||||
// CreatePartition Create Partition
|
||||
@ -268,3 +316,9 @@ func (mc *MilvusClient) Get(ctx context.Context, option client.QueryOption, call
|
||||
resultSet, err := mc.mClient.Get(ctx, option, callOptions...)
|
||||
return resultSet, err
|
||||
}
|
||||
|
||||
// HybridSearch hybrid search from collection
|
||||
func (mc *MilvusClient) HybridSearch(ctx context.Context, option client.HybridSearchOption, callOptions ...grpc.CallOption) ([]client.ResultSet, error) {
|
||||
resultSets, err := mc.mClient.HybridSearch(ctx, option, callOptions...)
|
||||
return resultSets, err
|
||||
}
|
||||
|
||||
@ -4,33 +4,38 @@ import "github.com/milvus-io/milvus/client/v2/index"
|
||||
|
||||
// cost default field name
|
||||
const (
|
||||
DefaultInt8FieldName = "int8"
|
||||
DefaultInt16FieldName = "int16"
|
||||
DefaultInt32FieldName = "int32"
|
||||
DefaultInt64FieldName = "int64"
|
||||
DefaultBoolFieldName = "bool"
|
||||
DefaultFloatFieldName = "float"
|
||||
DefaultDoubleFieldName = "double"
|
||||
DefaultVarcharFieldName = "varchar"
|
||||
DefaultJSONFieldName = "json"
|
||||
DefaultArrayFieldName = "array"
|
||||
DefaultFloatVecFieldName = "floatVec"
|
||||
DefaultBinaryVecFieldName = "binaryVec"
|
||||
DefaultFloat16VecFieldName = "fp16Vec"
|
||||
DefaultBFloat16VecFieldName = "bf16Vec"
|
||||
DefaultSparseVecFieldName = "sparseVec"
|
||||
DefaultDynamicNumberField = "dynamicNumber"
|
||||
DefaultDynamicStringField = "dynamicString"
|
||||
DefaultDynamicBoolField = "dynamicBool"
|
||||
DefaultDynamicListField = "dynamicList"
|
||||
DefaultBoolArrayField = "boolArray"
|
||||
DefaultInt8ArrayField = "int8Array"
|
||||
DefaultInt16ArrayField = "int16Array"
|
||||
DefaultInt32ArrayField = "int32Array"
|
||||
DefaultInt64ArrayField = "int64Array"
|
||||
DefaultFloatArrayField = "floatArray"
|
||||
DefaultDoubleArrayField = "doubleArray"
|
||||
DefaultVarcharArrayField = "varcharArray"
|
||||
DefaultInt8FieldName = "int8"
|
||||
DefaultInt16FieldName = "int16"
|
||||
DefaultInt32FieldName = "int32"
|
||||
DefaultInt64FieldName = "int64"
|
||||
DefaultBoolFieldName = "bool"
|
||||
DefaultFloatFieldName = "float"
|
||||
DefaultDoubleFieldName = "double"
|
||||
DefaultTextFieldName = "text"
|
||||
DefaultVarcharFieldName = "varchar"
|
||||
DefaultJSONFieldName = "json"
|
||||
DefaultArrayFieldName = "array"
|
||||
DefaultFloatVecFieldName = "floatVec"
|
||||
DefaultBinaryVecFieldName = "binaryVec"
|
||||
DefaultFloat16VecFieldName = "fp16Vec"
|
||||
DefaultBFloat16VecFieldName = "bf16Vec"
|
||||
DefaultTextSparseVecFieldName = "textSparseVec"
|
||||
DefaultSparseVecFieldName = "sparseVec"
|
||||
DefaultDynamicNumberField = "dynamicNumber"
|
||||
DefaultDynamicStringField = "dynamicString"
|
||||
DefaultDynamicBoolField = "dynamicBool"
|
||||
DefaultDynamicListField = "dynamicList"
|
||||
DefaultBoolArrayField = "boolArray"
|
||||
DefaultInt8ArrayField = "int8Array"
|
||||
DefaultInt16ArrayField = "int16Array"
|
||||
DefaultInt32ArrayField = "int32Array"
|
||||
DefaultInt64ArrayField = "int64Array"
|
||||
DefaultFloatArrayField = "floatArray"
|
||||
DefaultDoubleArrayField = "doubleArray"
|
||||
DefaultVarcharArrayField = "varcharArray"
|
||||
|
||||
DefaultFastPk = "id"
|
||||
DefaultFastVector = "vector"
|
||||
)
|
||||
|
||||
// cost for test cases
|
||||
@ -76,3 +81,20 @@ const (
|
||||
IndexStateFailed index.IndexState = 4
|
||||
IndexStateRetry index.IndexState = 5
|
||||
)
|
||||
|
||||
// properties
|
||||
const (
|
||||
CollectionTTLSeconds = "collection.ttl.seconds"
|
||||
MmapEnabled = "mmap.enabled"
|
||||
DatabaseMaxCollections = "database.max.collections"
|
||||
DatabaseResourceGroups = "database.resource_groups"
|
||||
DatabaseReplicaNumber = "database.replica.number"
|
||||
DatabaseForceDenyWriting = "database.force.deny.writing"
|
||||
DatabaseForceDenyReading = "database.force.deny.reading"
|
||||
DatabaseDiskQuotaMb = "database.diskQuota.mb"
|
||||
)
|
||||
|
||||
// const for full text search
|
||||
const (
|
||||
DefaultTextLang = "en"
|
||||
)
|
||||
|
||||
@ -5,6 +5,7 @@ import (
|
||||
"fmt"
|
||||
"math"
|
||||
"math/rand"
|
||||
"reflect"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
@ -153,3 +154,63 @@ var InvalidExpressions = []InvalidExprStruct{
|
||||
{Expr: fmt.Sprintf("%s[-1] > %d", DefaultInt8ArrayField, TestCapacity), ErrNil: false, ErrMsg: "cannot parse expression"}, // array[-1] >
|
||||
{Expr: fmt.Sprintf("%s[-1] > 1", DefaultJSONFieldName), ErrNil: false, ErrMsg: "invalid expression"}, // json[-1] >
|
||||
}
|
||||
|
||||
// Language constants for text generation
|
||||
const (
|
||||
English = "en"
|
||||
Chinese = "zh"
|
||||
)
|
||||
|
||||
func GenText(lang string) string {
|
||||
englishTopics := []string{
|
||||
"information retrieval", "data mining", "machine learning",
|
||||
"natural language processing", "text analysis", "search engines",
|
||||
"document indexing", "query processing", "relevance ranking",
|
||||
"semantic search",
|
||||
}
|
||||
englishVerbs := []string{
|
||||
"is", "focuses on", "deals with", "involves", "combines",
|
||||
"utilizes", "improves", "enables", "enhances", "supports",
|
||||
}
|
||||
englishObjects := []string{
|
||||
"large datasets", "text documents", "user queries", "search results",
|
||||
"information needs", "relevance scores", "ranking algorithms",
|
||||
"index structures", "query expansion", "document collections",
|
||||
}
|
||||
|
||||
chineseTopics := []string{
|
||||
"信息检索", "数据挖掘", "机器学习",
|
||||
"自然语言处理", "文本分析", "搜索引擎",
|
||||
"文档索引", "查询处理", "相关性排序",
|
||||
"语义搜索",
|
||||
}
|
||||
chineseVerbs := []string{
|
||||
"是", "专注于", "处理", "涉及", "结合",
|
||||
"利用", "改进", "实现", "提升", "支持",
|
||||
}
|
||||
chineseObjects := []string{
|
||||
"大规模数据集", "文本文档", "用户查询", "搜索结果",
|
||||
"信息需求", "相关性分数", "排序算法",
|
||||
"索引结构", "查询扩展", "文档集合",
|
||||
}
|
||||
|
||||
var topic, verb, object string
|
||||
switch lang {
|
||||
case English:
|
||||
topic = englishTopics[rand.Intn(len(englishTopics))]
|
||||
verb = englishVerbs[rand.Intn(len(englishVerbs))]
|
||||
object = englishObjects[rand.Intn(len(englishObjects))]
|
||||
return fmt.Sprintf("%s %s %s", topic, verb, object)
|
||||
case Chinese:
|
||||
topic = chineseTopics[rand.Intn(len(chineseTopics))]
|
||||
verb = chineseVerbs[rand.Intn(len(chineseVerbs))]
|
||||
object = chineseObjects[rand.Intn(len(chineseObjects))]
|
||||
return fmt.Sprintf("%s%s%s", topic, verb, object)
|
||||
default:
|
||||
return "Unsupported language"
|
||||
}
|
||||
}
|
||||
|
||||
func IsZeroValue(value interface{}) bool {
|
||||
return reflect.DeepEqual(value, reflect.Zero(reflect.TypeOf(value)).Interface())
|
||||
}
|
||||
|
||||
@ -1,5 +1,3 @@
|
||||
///go:build L0
|
||||
|
||||
package testcases
|
||||
|
||||
import (
|
||||
|
||||
@ -2,6 +2,7 @@ package testcases
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -9,6 +10,7 @@ import (
|
||||
"go.uber.org/zap"
|
||||
|
||||
"github.com/milvus-io/milvus/client/v2/entity"
|
||||
"github.com/milvus-io/milvus/client/v2/index"
|
||||
client "github.com/milvus-io/milvus/client/v2/milvusclient"
|
||||
"github.com/milvus-io/milvus/pkg/log"
|
||||
"github.com/milvus-io/milvus/tests/go_client/common"
|
||||
@ -40,6 +42,74 @@ func TestCreateCollection(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// fast: create -> index -> load
|
||||
func TestCreateCollectionFast(t *testing.T) {
|
||||
// test collection property mmap
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
// create collection option: WithConsistencyLevel Strong,
|
||||
collName := common.GenRandomString("alter", 6)
|
||||
err := mc.CreateCollection(ctx, client.SimpleCreateCollectionOptions(collName, common.DefaultDim))
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
// verify collection option
|
||||
coll, _ := mc.DescribeCollection(ctx, client.NewDescribeCollectionOption(collName))
|
||||
require.Equal(t, entity.FieldTypeInt64, coll.Schema.PKField().DataType)
|
||||
|
||||
// load -> insert
|
||||
prepare, _ := hp.CollPrepare.InsertData(ctx, t, mc, hp.NewInsertParams(coll.Schema), hp.TNewDataOption())
|
||||
prepare.FlushData(ctx, t, mc, collName)
|
||||
|
||||
countRes, err := mc.Query(ctx, client.NewQueryOption(collName).WithFilter("").WithOutputFields(common.QueryCountFieldName))
|
||||
common.CheckErr(t, err, true)
|
||||
count, _ := countRes.Fields[0].GetAsInt64(0)
|
||||
require.EqualValues(t, common.DefaultNb, count)
|
||||
|
||||
vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector)
|
||||
resSearch, err := mc.Search(ctx, client.NewSearchOption(collName, common.DefaultLimit, vectors))
|
||||
common.CheckErr(t, err, true)
|
||||
common.CheckSearchResult(t, resSearch, common.DefaultNq, common.DefaultLimit)
|
||||
}
|
||||
|
||||
func TestCreateCollectionFastOption(t *testing.T) {
|
||||
// test create collection fast with option: ConsistencyLevel, varcharPk, indexOption
|
||||
// Collection AutoID not works !!!, please set it on the field side~
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
// create collection option: WithConsistencyLevel Strong,
|
||||
collName := common.GenRandomString("alter", 6)
|
||||
index := index.NewHNSWIndex(entity.COSINE, 8, 96)
|
||||
indexOption := client.NewCreateIndexOption(collName, common.DefaultFastVector, index)
|
||||
err := mc.CreateCollection(ctx, client.SimpleCreateCollectionOptions(collName, common.DefaultDim).WithDynamicSchema(true).
|
||||
WithConsistencyLevel(entity.ClStrong).WithIndexOptions(indexOption).WithVarcharPK(true, 10))
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
// verify collection option
|
||||
coll, _ := mc.DescribeCollection(ctx, client.NewDescribeCollectionOption(collName))
|
||||
require.Equal(t, entity.ClStrong, coll.ConsistencyLevel)
|
||||
require.Equal(t, entity.FieldTypeVarChar, coll.Schema.PKField().DataType)
|
||||
t.Log("https://github.com/milvus-io/milvus/issues/39524")
|
||||
// descIdx, _ := mc.DescribeIndex(ctx, client.NewDescribeIndexOption(collName, common.DefaultFastVector))
|
||||
// common.CheckIndex(t, descIdx, index, common.TNewCheckIndexOpt(common.DefaultNb))
|
||||
|
||||
// insert
|
||||
hp.CollPrepare.InsertData(ctx, t, mc, hp.NewInsertParams(coll.Schema), hp.TNewDataOption())
|
||||
|
||||
countRes, err := mc.Query(ctx, client.NewQueryOption(collName).
|
||||
WithFilter("").WithOutputFields(common.QueryCountFieldName))
|
||||
common.CheckErr(t, err, true)
|
||||
count, _ := countRes.Fields[0].GetAsInt64(0)
|
||||
require.EqualValues(t, common.DefaultNb, count)
|
||||
|
||||
vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector)
|
||||
resSearch, err := mc.Search(ctx, client.NewSearchOption(collName, common.DefaultLimit, vectors).WithOutputFields(common.DefaultDynamicNumberField))
|
||||
common.CheckErr(t, err, true)
|
||||
common.CheckSearchResult(t, resSearch, common.DefaultNq, common.DefaultLimit)
|
||||
common.CheckOutputFields(t, []string{common.DefaultDynamicNumberField}, resSearch[0].Fields)
|
||||
}
|
||||
|
||||
func TestCreateAutoIdCollectionField(t *testing.T) {
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
@ -122,7 +192,7 @@ func TestCreateAutoIdCollectionSchema(t *testing.T) {
|
||||
|
||||
// test create auto collection with collection option
|
||||
func TestCreateAutoIdCollection(t *testing.T) {
|
||||
t.Skip("waiting for valid AutoId from collection option")
|
||||
t.Skip("https://github.com/milvus-io/milvus/issues/39523")
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
@ -132,7 +202,7 @@ func TestCreateAutoIdCollection(t *testing.T) {
|
||||
pkField := entity.NewField().WithName("pk").WithDataType(pkFieldType).WithIsPrimaryKey(true).WithMaxLength(common.MaxLength)
|
||||
|
||||
// pk field with name
|
||||
schema := entity.NewSchema().WithName(collName).WithField(pkField).WithField(vecField)
|
||||
schema := entity.NewSchema().WithName(collName).WithField(pkField).WithField(vecField).WithAutoID(true)
|
||||
err := mc.CreateCollection(ctx, client.NewCreateCollectionOption(collName, schema).WithAutoID(true))
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
@ -777,7 +847,7 @@ func TestCreateVectorWithoutDim(t *testing.T) {
|
||||
entity.NewField().WithName(vecFieldName).WithDataType(entity.FieldTypeFloatVector),
|
||||
).WithName(collName)
|
||||
err := mc.CreateCollection(ctx, client.NewCreateCollectionOption(collName, schema))
|
||||
common.CheckErr(t, err, false, fmt.Sprintf("dimension is not defined in field type params of field %s, check type param `dim` for vector field", vecFieldName))
|
||||
common.CheckErr(t, err, false, "dimension is not defined in field type params")
|
||||
}
|
||||
|
||||
// specify dim for sparse vector -> error
|
||||
@ -838,7 +908,7 @@ func TestCreateVarcharArrayInvalidLength(t *testing.T) {
|
||||
for _, invalidLength := range []int64{-1, 0, common.MaxLength + 1} {
|
||||
arrayVarcharField.WithMaxLength(invalidLength)
|
||||
err := mc.CreateCollection(ctx, client.NewCreateCollectionOption(collName, schema))
|
||||
common.CheckErr(t, err, false, fmt.Sprintf("the maximum length specified for a VarChar field(%s) should be in (0, 65535], but got %d instead: invalid parameter", arrayVarcharField.Name, invalidLength))
|
||||
common.CheckErr(t, err, false, "the maximum length specified for a VarChar field(array) should be in (0, 65535]")
|
||||
}
|
||||
}
|
||||
|
||||
@ -860,7 +930,7 @@ func TestCreateVarcharInvalidLength(t *testing.T) {
|
||||
for _, invalidLength := range []int64{-1, 0, common.MaxLength + 1} {
|
||||
varcharField.WithMaxLength(invalidLength)
|
||||
err := mc.CreateCollection(ctx, client.NewCreateCollectionOption(collName, schema))
|
||||
common.CheckErr(t, err, false, fmt.Sprintf("the maximum length specified for a VarChar field(%s) should be in (0, 65535], but got %d instead", varcharField.Name, invalidLength))
|
||||
common.CheckErr(t, err, false, "the maximum length specified for a VarChar field(varchar) should be in (0, 65535]")
|
||||
}
|
||||
}
|
||||
|
||||
@ -934,3 +1004,237 @@ func TestCreateCollectionInvalid(t *testing.T) {
|
||||
common.CheckErr(t, err, false, mSchema.errMsg)
|
||||
}
|
||||
}
|
||||
|
||||
// test rename collection
|
||||
func TestRenameCollection(t *testing.T) {
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
// create collection
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
|
||||
|
||||
// rename collection and verify
|
||||
newName := common.GenRandomString("new", 6)
|
||||
err := mc.RenameCollection(ctx, client.NewRenameCollectionOption(schema.CollectionName, newName))
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
collections, _ := mc.ListCollections(ctx, client.NewListCollectionOption())
|
||||
require.Contains(t, collections, newName)
|
||||
require.NotContains(t, collections, schema.CollectionName)
|
||||
|
||||
_, err = mc.ListIndexes(ctx, client.NewListIndexOption(schema.CollectionName))
|
||||
common.CheckErr(t, err, false, "collection not found")
|
||||
|
||||
schema.CollectionName = newName
|
||||
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption())
|
||||
prepare.FlushData(ctx, t, mc, schema.CollectionName)
|
||||
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
|
||||
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
|
||||
|
||||
vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector)
|
||||
resSearch, errSearch := mc.Search(ctx, client.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors))
|
||||
common.CheckErr(t, errSearch, true)
|
||||
common.CheckSearchResult(t, resSearch, common.DefaultNq, common.DefaultLimit)
|
||||
|
||||
stats, err := mc.GetCollectionStats(ctx, client.NewGetCollectionStatsOption(newName))
|
||||
common.CheckErr(t, err, true)
|
||||
require.Equal(t, map[string]string{common.RowCount: strconv.Itoa(common.DefaultNb)}, stats)
|
||||
}
|
||||
|
||||
// There are collections with the same name in different db. Rename one of them.
|
||||
func TestRenameCollectionDb(t *testing.T) {
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
// create collection
|
||||
collectionName := common.GenRandomString("re", 6)
|
||||
mc.CreateCollection(ctx, client.SimpleCreateCollectionOptions(collectionName, common.DefaultDim))
|
||||
|
||||
// create a database and use database
|
||||
dbName := common.GenRandomString("db", 4)
|
||||
mc.CreateDatabase(ctx, client.NewCreateDatabaseOption(dbName))
|
||||
mc.UseDatabase(ctx, client.NewUseDatabaseOption(dbName))
|
||||
mc.CreateCollection(ctx, client.SimpleCreateCollectionOptions(collectionName, common.DefaultDim))
|
||||
|
||||
// rename db collection rather than default db collection
|
||||
newName := common.GenRandomString("new", 6)
|
||||
err := mc.RenameCollection(ctx, client.NewRenameCollectionOption(collectionName, newName))
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
collections, _ := mc.ListCollections(ctx, client.NewListCollectionOption())
|
||||
require.Contains(t, collections, newName)
|
||||
require.NotContains(t, collections, collectionName)
|
||||
|
||||
// verify default db collection
|
||||
mc.UseDatabase(ctx, client.NewUseDatabaseOption(common.DefaultDb))
|
||||
collectionsDefault, _ := mc.ListCollections(ctx, client.NewListCollectionOption())
|
||||
require.Contains(t, collectionsDefault, collectionName)
|
||||
require.NotContains(t, collectionsDefault, newName)
|
||||
}
|
||||
|
||||
func TestRenameCollectionInvalidName(t *testing.T) {
|
||||
// connect
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
// create collection
|
||||
collectionName := common.GenRandomString("re", 6)
|
||||
mc.CreateCollection(ctx, client.SimpleCreateCollectionOptions(collectionName, common.DefaultDim))
|
||||
|
||||
// rename collection with invalid name
|
||||
for _, invalidName := range common.GenInvalidNames() {
|
||||
log.Debug("TestCreateCollectionWithInvalidFieldName", zap.String("fieldName", invalidName))
|
||||
err := mc.RenameCollection(ctx, client.NewRenameCollectionOption(collectionName, invalidName))
|
||||
common.CheckErr(t, err, false, "collection name should not be empty",
|
||||
"the first character of a collection name must be an underscore or letter",
|
||||
"collection name can only contain numbers, letters and underscores",
|
||||
"the length of a collection name must be less than 255 characters",
|
||||
"collection name can only contain numbers, letters, and underscores")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRenameCollectionAdvanced(t *testing.T) {
|
||||
// connect
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
// create 2 collections
|
||||
name1 := common.GenRandomString("name1", 6)
|
||||
name2 := common.GenRandomString("name2", 6)
|
||||
mc.CreateCollection(ctx, client.SimpleCreateCollectionOptions(name1, common.DefaultDim))
|
||||
mc.CreateCollection(ctx, client.SimpleCreateCollectionOptions(name2, common.DefaultDim))
|
||||
|
||||
// rename: old name same with new name
|
||||
err := mc.RenameCollection(ctx, client.NewRenameCollectionOption(name1, name1))
|
||||
common.CheckErr(t, err, false, "duplicated new collection name")
|
||||
|
||||
// rename to a existed name
|
||||
err = mc.RenameCollection(ctx, client.NewRenameCollectionOption(name1, name2))
|
||||
common.CheckErr(t, err, false, "duplicated new collection name")
|
||||
|
||||
// rename a not existed collection
|
||||
err = mc.RenameCollection(ctx, client.NewRenameCollectionOption(common.GenRandomString("a", 2), common.GenRandomString("b", 2)))
|
||||
common.CheckErr(t, err, false, "collection not found")
|
||||
}
|
||||
|
||||
// alter collection ttl property
|
||||
func TestCollectionPropertyTtl(t *testing.T) {
|
||||
// test collection property ttl
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
// create collection
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
|
||||
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption())
|
||||
prepare.FlushData(ctx, t, mc, schema.CollectionName)
|
||||
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
|
||||
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
|
||||
res, _ := mc.Query(ctx, client.NewQueryOption(schema.CollectionName).WithFilter("").WithOutputFields(common.QueryCountFieldName))
|
||||
countBefore, _ := res.GetColumn(common.QueryCountFieldName).GetAsInt64(0)
|
||||
require.EqualValues(t, common.DefaultNb, countBefore)
|
||||
|
||||
err := mc.AlterCollectionProperties(ctx, client.NewAlterCollectionPropertiesOption(schema.CollectionName).WithProperty(common.CollectionTTLSeconds, 2))
|
||||
common.CheckErr(t, err, true)
|
||||
coll, _ := mc.DescribeCollection(ctx, client.NewDescribeCollectionOption(schema.CollectionName))
|
||||
require.Equal(t, map[string]string{common.CollectionTTLSeconds: "2"}, coll.Properties)
|
||||
|
||||
time.Sleep(5 * time.Second)
|
||||
|
||||
res, _ = mc.Query(ctx, client.NewQueryOption(schema.CollectionName).WithFilter("").WithOutputFields(common.QueryCountFieldName))
|
||||
countAfter, _ := res.GetColumn(common.QueryCountFieldName).GetAsInt64(0)
|
||||
require.Contains(t, []int64{0, int64(common.DefaultNb)}, countAfter)
|
||||
|
||||
err = mc.DropCollectionProperties(ctx, client.NewDropCollectionPropertiesOption(schema.CollectionName, common.CollectionTTLSeconds))
|
||||
common.CheckErr(t, err, true)
|
||||
coll, _ = mc.DescribeCollection(ctx, client.NewDescribeCollectionOption(schema.CollectionName))
|
||||
require.Equal(t, map[string]string{}, coll.Properties)
|
||||
}
|
||||
|
||||
// create collection with property -> alter property -> writing and reading
|
||||
func TestCollectionWithPropertyAlterMmap(t *testing.T) {
|
||||
// test collection property mmap
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
// create collection
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec),
|
||||
hp.TNewFieldsOption(), hp.TNewSchemaOption(), hp.TWithProperties(map[string]any{common.MmapEnabled: false}))
|
||||
|
||||
coll, _ := mc.DescribeCollection(ctx, client.NewDescribeCollectionOption(schema.CollectionName))
|
||||
require.Equal(t, map[string]string{common.MmapEnabled: "false"}, coll.Properties)
|
||||
log.Info("TestCollectionPropertyMmap.DescribeCollection", zap.Any("properties", coll.Properties))
|
||||
|
||||
// alter properties
|
||||
err := mc.AlterCollectionProperties(ctx, client.NewAlterCollectionPropertiesOption(schema.CollectionName).WithProperty(common.MmapEnabled, true))
|
||||
common.CheckErr(t, err, true)
|
||||
coll, _ = mc.DescribeCollection(ctx, client.NewDescribeCollectionOption(schema.CollectionName))
|
||||
require.Equal(t, map[string]string{common.MmapEnabled: "true"}, coll.Properties)
|
||||
|
||||
// writing and reading
|
||||
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption())
|
||||
prepare.FlushData(ctx, t, mc, schema.CollectionName)
|
||||
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
|
||||
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
|
||||
res, _ := mc.Query(ctx, client.NewQueryOption(schema.CollectionName).WithFilter("").WithOutputFields(common.QueryCountFieldName))
|
||||
countBefore, _ := res.GetColumn(common.QueryCountFieldName).GetAsInt64(0)
|
||||
require.EqualValues(t, common.DefaultNb, countBefore)
|
||||
}
|
||||
|
||||
func TestCollectionPropertyMmap(t *testing.T) {
|
||||
// test collection property mmap
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
// create collection
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec),
|
||||
hp.TNewFieldsOption(), hp.TNewSchemaOption())
|
||||
|
||||
// alter properties
|
||||
err := mc.AlterCollectionProperties(ctx, client.NewAlterCollectionPropertiesOption(schema.CollectionName).WithProperty(common.MmapEnabled, true))
|
||||
common.CheckErr(t, err, true)
|
||||
coll, _ := mc.DescribeCollection(ctx, client.NewDescribeCollectionOption(schema.CollectionName))
|
||||
require.Equal(t, map[string]string{common.MmapEnabled: "true"}, coll.Properties)
|
||||
|
||||
// writing and reading
|
||||
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption())
|
||||
prepare.FlushData(ctx, t, mc, schema.CollectionName)
|
||||
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
|
||||
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
|
||||
res, _ := mc.Query(ctx, client.NewQueryOption(schema.CollectionName).WithFilter("").WithOutputFields(common.QueryCountFieldName))
|
||||
countBefore, _ := res.GetColumn(common.QueryCountFieldName).GetAsInt64(0)
|
||||
require.EqualValues(t, common.DefaultNb, countBefore)
|
||||
|
||||
err = mc.DropCollectionProperties(ctx, client.NewDropCollectionPropertiesOption(schema.CollectionName, common.MmapEnabled))
|
||||
common.CheckErr(t, err, false, "can not delete mmap properties if collection loaded")
|
||||
|
||||
// release collection and drop property
|
||||
mc.ReleaseCollection(ctx, client.NewReleaseCollectionOption(schema.CollectionName))
|
||||
err = mc.DropCollectionProperties(ctx, client.NewDropCollectionPropertiesOption(schema.CollectionName, common.MmapEnabled))
|
||||
common.CheckErr(t, err, true)
|
||||
coll, _ = mc.DescribeCollection(ctx, client.NewDescribeCollectionOption(schema.CollectionName))
|
||||
require.Equal(t, map[string]string{}, coll.Properties)
|
||||
}
|
||||
|
||||
func TestCollectionFakeProperties(t *testing.T) {
|
||||
// test collection property mmap
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
// create collection with fake property
|
||||
collName := common.GenRandomString("alter", 6)
|
||||
err := mc.CreateCollection(ctx, client.SimpleCreateCollectionOptions(collName, common.DefaultDim).WithProperty("1", "bbb"))
|
||||
common.CheckErr(t, err, true)
|
||||
coll, _ := mc.DescribeCollection(ctx, client.NewDescribeCollectionOption(collName))
|
||||
require.Equal(t, map[string]string{"1": "bbb"}, coll.Properties)
|
||||
|
||||
// alter collection with fake property
|
||||
err = mc.AlterCollectionProperties(ctx, client.NewAlterCollectionPropertiesOption(collName).WithProperty("2", 1))
|
||||
common.CheckErr(t, err, true)
|
||||
coll, _ = mc.DescribeCollection(ctx, client.NewDescribeCollectionOption(collName))
|
||||
require.Equal(t, map[string]string{"1": "bbb", "2": "1"}, coll.Properties)
|
||||
|
||||
err = mc.DropCollectionProperties(ctx, client.NewDropCollectionPropertiesOption(collName, "ccc"))
|
||||
common.CheckErr(t, err, true)
|
||||
coll, _ = mc.DescribeCollection(ctx, client.NewDescribeCollectionOption(collName))
|
||||
require.Equal(t, map[string]string{"1": "bbb", "2": "1"}, coll.Properties)
|
||||
}
|
||||
|
||||
@ -2,6 +2,7 @@ package testcases
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -23,10 +24,10 @@ func teardownTest(t *testing.T) func(t *testing.T) {
|
||||
// drop all db
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
dbs, _ := mc.ListDatabases(ctx, client.NewListDatabaseOption())
|
||||
dbs, _ := mc.ListDatabase(ctx, client.NewListDatabaseOption())
|
||||
for _, db := range dbs {
|
||||
if db != common.DefaultDb {
|
||||
_ = mc.UsingDatabase(ctx, client.NewUseDatabaseOption(db))
|
||||
_ = mc.UseDatabase(ctx, client.NewUseDatabaseOption(db))
|
||||
collections, _ := mc.ListCollections(ctx, client.NewListCollectionOption())
|
||||
for _, coll := range collections {
|
||||
_ = mc.DropCollection(ctx, client.NewDropCollectionOption(coll))
|
||||
@ -50,15 +51,12 @@ func TestDatabase(t *testing.T) {
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
// list db and verify db1 in dbs
|
||||
dbs, errList := clientDefault.ListDatabases(ctx, client.NewListDatabaseOption())
|
||||
dbs, errList := clientDefault.ListDatabase(ctx, client.NewListDatabaseOption())
|
||||
common.CheckErr(t, errList, true)
|
||||
require.Containsf(t, dbs, dbName1, fmt.Sprintf("%s db not in dbs: %v", dbName1, dbs))
|
||||
|
||||
// new client with db1 -> using db
|
||||
// new client with db1
|
||||
clientDB1 := createMilvusClient(ctx, t, &client.ClientConfig{Address: *addr, DBName: dbName1})
|
||||
t.Log("https://github.com/milvus-io/milvus/issues/34137")
|
||||
err = clientDB1.UsingDatabase(ctx, client.NewUseDatabaseOption(dbName1))
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
// create collections -> verify collections contains
|
||||
_, db1Col1 := hp.CollPrepare.CreateCollection(ctx, t, clientDB1, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
|
||||
@ -72,24 +70,24 @@ func TestDatabase(t *testing.T) {
|
||||
dbName2 := common.GenRandomString("db2", 4)
|
||||
err = clientDefault.CreateDatabase(ctx, client.NewCreateDatabaseOption(dbName2))
|
||||
common.CheckErr(t, err, true)
|
||||
dbs, err = clientDefault.ListDatabases(ctx, client.NewListDatabaseOption())
|
||||
dbs, err = clientDefault.ListDatabase(ctx, client.NewListDatabaseOption())
|
||||
common.CheckErr(t, err, true)
|
||||
require.Containsf(t, dbs, dbName2, fmt.Sprintf("%s db not in dbs: %v", dbName2, dbs))
|
||||
|
||||
// using db2 -> create collection -> drop collection
|
||||
err = clientDefault.UsingDatabase(ctx, client.NewUseDatabaseOption(dbName2))
|
||||
err = clientDefault.UseDatabase(ctx, client.NewUseDatabaseOption(dbName2))
|
||||
common.CheckErr(t, err, true)
|
||||
_, db2Col1 := hp.CollPrepare.CreateCollection(ctx, t, clientDefault, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
|
||||
err = clientDefault.DropCollection(ctx, client.NewDropCollectionOption(db2Col1.CollectionName))
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
// using empty db -> drop db2
|
||||
clientDefault.UsingDatabase(ctx, client.NewUseDatabaseOption(""))
|
||||
clientDefault.UseDatabase(ctx, client.NewUseDatabaseOption(""))
|
||||
err = clientDefault.DropDatabase(ctx, client.NewDropDatabaseOption(dbName2))
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
// list db and verify db drop success
|
||||
dbs, err = clientDefault.ListDatabases(ctx, client.NewListDatabaseOption())
|
||||
dbs, err = clientDefault.ListDatabase(ctx, client.NewListDatabaseOption())
|
||||
common.CheckErr(t, err, true)
|
||||
require.NotContains(t, dbs, dbName2)
|
||||
|
||||
@ -98,7 +96,7 @@ func TestDatabase(t *testing.T) {
|
||||
common.CheckErr(t, err, false, "must drop all collections before drop database")
|
||||
|
||||
// drop all db1's collections -> drop db1
|
||||
clientDB1.UsingDatabase(ctx, client.NewUseDatabaseOption(dbName1))
|
||||
clientDB1.UseDatabase(ctx, client.NewUseDatabaseOption(dbName1))
|
||||
err = clientDB1.DropCollection(ctx, client.NewDropCollectionOption(db1Col1.CollectionName))
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
@ -112,7 +110,7 @@ func TestDatabase(t *testing.T) {
|
||||
err = clientDefault.DropDatabase(ctx, client.NewDropDatabaseOption(common.DefaultDb))
|
||||
common.CheckErr(t, err, false, "can not drop default database")
|
||||
|
||||
dbs, err = clientDefault.ListDatabases(ctx, client.NewListDatabaseOption())
|
||||
dbs, err = clientDefault.ListDatabase(ctx, client.NewListDatabaseOption())
|
||||
common.CheckErr(t, err, true)
|
||||
require.Containsf(t, dbs, common.DefaultDb, fmt.Sprintf("The db %s not in: %v", common.DefaultDb, dbs))
|
||||
}
|
||||
@ -160,7 +158,7 @@ func TestDropDb(t *testing.T) {
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
// using db and drop the db
|
||||
err = mc.UsingDatabase(ctx, client.NewUseDatabaseOption(dbName))
|
||||
err = mc.UseDatabase(ctx, client.NewUseDatabaseOption(dbName))
|
||||
common.CheckErr(t, err, true)
|
||||
err = mc.DropDatabase(ctx, client.NewDropDatabaseOption(dbName))
|
||||
common.CheckErr(t, err, true)
|
||||
@ -170,7 +168,7 @@ func TestDropDb(t *testing.T) {
|
||||
common.CheckErr(t, err, false, fmt.Sprintf("database not found[database=%s]", dbName))
|
||||
|
||||
// using default db and verify collections
|
||||
err = mc.UsingDatabase(ctx, client.NewUseDatabaseOption(common.DefaultDb))
|
||||
err = mc.UseDatabase(ctx, client.NewUseDatabaseOption(common.DefaultDb))
|
||||
common.CheckErr(t, err, true)
|
||||
collections, _ = mc.ListCollections(ctx, listCollOpt)
|
||||
require.Contains(t, collections, defCol.CollectionName)
|
||||
@ -205,24 +203,23 @@ func TestUsingDb(t *testing.T) {
|
||||
|
||||
// using not existed db
|
||||
dbName := common.GenRandomString("db", 4)
|
||||
err := mc.UsingDatabase(ctx, client.NewUseDatabaseOption(dbName))
|
||||
err := mc.UseDatabase(ctx, client.NewUseDatabaseOption(dbName))
|
||||
common.CheckErr(t, err, false, fmt.Sprintf("database not found[database=%s]", dbName))
|
||||
|
||||
// using empty db
|
||||
err = mc.UsingDatabase(ctx, client.NewUseDatabaseOption(""))
|
||||
err = mc.UseDatabase(ctx, client.NewUseDatabaseOption(""))
|
||||
common.CheckErr(t, err, true)
|
||||
collections, _ = mc.ListCollections(ctx, listCollOpt)
|
||||
require.Contains(t, collections, col.CollectionName)
|
||||
|
||||
// using current db
|
||||
err = mc.UsingDatabase(ctx, client.NewUseDatabaseOption(common.DefaultDb))
|
||||
err = mc.UseDatabase(ctx, client.NewUseDatabaseOption(common.DefaultDb))
|
||||
common.CheckErr(t, err, true)
|
||||
collections, _ = mc.ListCollections(ctx, listCollOpt)
|
||||
require.Contains(t, collections, col.CollectionName)
|
||||
}
|
||||
|
||||
func TestClientWithDb(t *testing.T) {
|
||||
t.Skip("https://github.com/milvus-io/milvus/issues/34137")
|
||||
teardownSuite := teardownTest(t)
|
||||
defer teardownSuite(t)
|
||||
|
||||
@ -262,7 +259,7 @@ func TestClientWithDb(t *testing.T) {
|
||||
require.Containsf(t, dbCollections, dbCol1.CollectionName, fmt.Sprintf("The collection %s not in: %v", dbCol1.CollectionName, dbCollections))
|
||||
|
||||
// using default db and collection not in
|
||||
_ = mcDb.UsingDatabase(ctx, client.NewUseDatabaseOption(common.DefaultDb))
|
||||
_ = mcDb.UseDatabase(ctx, client.NewUseDatabaseOption(common.DefaultDb))
|
||||
defCollections, _ = mcDb.ListCollections(ctx, listCollOpt)
|
||||
require.NotContains(t, defCollections, dbCol1.CollectionName)
|
||||
|
||||
@ -276,6 +273,166 @@ func TestClientWithDb(t *testing.T) {
|
||||
require.Contains(t, defCollections, defCol1.CollectionName)
|
||||
}
|
||||
|
||||
func TestAlterDatabase(t *testing.T) {
|
||||
t.Skip("waiting for AlterDatabase and DescribeDatabase")
|
||||
func TestDatabasePropertiesCollectionsNum(t *testing.T) {
|
||||
// create db with properties
|
||||
teardownSuite := teardownTest(t)
|
||||
defer teardownSuite(t)
|
||||
|
||||
// create db
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
dbName := common.GenRandomString("db", 4)
|
||||
err := mc.CreateDatabase(ctx, client.NewCreateDatabaseOption(dbName))
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
// alter database properties
|
||||
maxCollections := 2
|
||||
err = mc.AlterDatabaseProperties(ctx, client.NewAlterDatabasePropertiesOption(dbName).WithProperty(common.DatabaseMaxCollections, maxCollections))
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
// describe database
|
||||
db, _ := mc.DescribeDatabase(ctx, client.NewDescribeDatabaseOption(dbName))
|
||||
require.Equal(t, map[string]string{common.DatabaseMaxCollections: strconv.Itoa(maxCollections)}, db.Properties)
|
||||
require.Equal(t, dbName, db.Name)
|
||||
|
||||
// verify properties works
|
||||
mc.UseDatabase(ctx, client.NewUseDatabaseOption(dbName))
|
||||
var collections []string
|
||||
for i := 0; i < maxCollections; i++ {
|
||||
_, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
|
||||
collections = append(collections, schema.CollectionName)
|
||||
}
|
||||
fields := hp.FieldsFact.GenFieldsForCollection(hp.Int64Vec, hp.TNewFieldsOption())
|
||||
schema := hp.GenSchema(hp.TNewSchemaOption().TWithFields(fields))
|
||||
err = mc.CreateCollection(ctx, client.NewCreateCollectionOption(schema.CollectionName, schema))
|
||||
common.CheckErr(t, err, false, "exceeded the limit number of collections")
|
||||
|
||||
// Other db are not restricted by this property
|
||||
mc.UseDatabase(ctx, client.NewUseDatabaseOption(common.DefaultDb))
|
||||
for i := 0; i < maxCollections+1; i++ {
|
||||
hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
|
||||
}
|
||||
|
||||
// drop properties
|
||||
mc.UseDatabase(ctx, client.NewUseDatabaseOption(dbName))
|
||||
errDrop := mc.DropDatabaseProperties(ctx, client.NewDropDatabasePropertiesOption(dbName, common.DatabaseMaxCollections))
|
||||
common.CheckErr(t, errDrop, true)
|
||||
_, schema1 := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
|
||||
collections = append(collections, schema1.CollectionName)
|
||||
|
||||
// verify collection num
|
||||
collectionsList, _ := mc.ListCollections(ctx, client.NewListCollectionOption())
|
||||
require.Subset(t, collectionsList, collections)
|
||||
require.GreaterOrEqual(t, len(collectionsList), maxCollections)
|
||||
|
||||
// describe database after drop properties
|
||||
db, _ = mc.DescribeDatabase(ctx, client.NewDescribeDatabaseOption(dbName))
|
||||
require.Equal(t, map[string]string{}, db.Properties)
|
||||
require.Equal(t, dbName, db.Name)
|
||||
}
|
||||
|
||||
func TestDatabasePropertiesRgReplicas(t *testing.T) {
|
||||
// create db with properties
|
||||
teardownSuite := teardownTest(t)
|
||||
defer teardownSuite(t)
|
||||
|
||||
// create db
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
dbName := common.GenRandomString("db", 4)
|
||||
err := mc.CreateDatabase(ctx, client.NewCreateDatabaseOption(dbName))
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
// alter database properties
|
||||
err = mc.AlterDatabaseProperties(ctx, client.NewAlterDatabasePropertiesOption(dbName).
|
||||
WithProperty(common.DatabaseResourceGroups, "rg1").WithProperty(common.DatabaseReplicaNumber, 2))
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
// describe database
|
||||
db, _ := mc.DescribeDatabase(ctx, client.NewDescribeDatabaseOption(dbName))
|
||||
require.Equal(t, map[string]string{common.DatabaseResourceGroups: "rg1", common.DatabaseReplicaNumber: "2"}, db.Properties)
|
||||
|
||||
mc.UseDatabase(ctx, client.NewUseDatabaseOption(dbName))
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true))
|
||||
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption().TWithNb(1000))
|
||||
prepare.FlushData(ctx, t, mc, schema.CollectionName)
|
||||
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
|
||||
|
||||
_, err = mc.LoadCollection(ctx, client.NewLoadCollectionOption(schema.CollectionName))
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
_, err = mc.Query(ctx, client.NewQueryOption(schema.CollectionName).WithLimit(10))
|
||||
common.CheckErr(t, err, true)
|
||||
}
|
||||
|
||||
func TestDatabasePropertyDeny(t *testing.T) {
|
||||
t.Skip("https://zilliz.atlassian.net/browse/VDC-7858")
|
||||
// create db with properties
|
||||
teardownSuite := teardownTest(t)
|
||||
defer teardownSuite(t)
|
||||
|
||||
// create db and use db
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
dbName := common.GenRandomString("db", 4)
|
||||
err := mc.CreateDatabase(ctx, client.NewCreateDatabaseOption(dbName))
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
// alter database properties and check
|
||||
err = mc.AlterDatabaseProperties(ctx, client.NewAlterDatabasePropertiesOption(dbName).
|
||||
WithProperty(common.DatabaseForceDenyWriting, true).
|
||||
WithProperty(common.DatabaseForceDenyReading, true))
|
||||
common.CheckErr(t, err, true)
|
||||
db, _ := mc.DescribeDatabase(ctx, client.NewDescribeDatabaseOption(dbName))
|
||||
require.Equal(t, map[string]string{common.DatabaseForceDenyWriting: "true", common.DatabaseForceDenyReading: "true"}, db.Properties)
|
||||
|
||||
err = mc.UseDatabase(ctx, client.NewUseDatabaseOption(dbName))
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
// prepare collection: create -> index -> load
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true))
|
||||
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
|
||||
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
|
||||
|
||||
// reading
|
||||
_, err = mc.Query(ctx, client.NewQueryOption(schema.CollectionName).WithLimit(10))
|
||||
common.CheckErr(t, err, false, "access has been disabled by the administrator")
|
||||
|
||||
// writing
|
||||
columns, _ := hp.GenColumnsBasedSchema(schema, hp.TNewDataOption().TWithNb(10))
|
||||
_, err = mc.Insert(ctx, client.NewColumnBasedInsertOption(schema.CollectionName, columns...))
|
||||
common.CheckErr(t, err, false, "access has been disabled by the administrator")
|
||||
}
|
||||
|
||||
func TestDatabaseFakeProperties(t *testing.T) {
|
||||
// create db with properties
|
||||
teardownSuite := teardownTest(t)
|
||||
defer teardownSuite(t)
|
||||
|
||||
// create db
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
dbName := common.GenRandomString("db", 4)
|
||||
err := mc.CreateDatabase(ctx, client.NewCreateDatabaseOption(dbName))
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
// alter database with useless properties
|
||||
properties := map[string]any{
|
||||
"key_1": 1,
|
||||
"key2": 1.9,
|
||||
"key-3": true,
|
||||
"key.4": "a.b.c",
|
||||
}
|
||||
for key, value := range properties {
|
||||
err = mc.AlterDatabaseProperties(ctx, client.NewAlterDatabasePropertiesOption(dbName).WithProperty(key, value))
|
||||
common.CheckErr(t, err, true)
|
||||
}
|
||||
|
||||
// describe database
|
||||
db, _ := mc.DescribeDatabase(ctx, client.NewDescribeDatabaseOption(dbName))
|
||||
require.EqualValues(t, map[string]string{"key_1": "1", "key2": "1.9", "key-3": "true", "key.4": "a.b.c"}, db.Properties)
|
||||
|
||||
// drop database properties
|
||||
err = mc.DropDatabaseProperties(ctx, client.NewDropDatabasePropertiesOption(dbName, "aaa"))
|
||||
common.CheckErr(t, err, true)
|
||||
}
|
||||
|
||||
@ -2,6 +2,7 @@ package testcases
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -440,7 +441,6 @@ func TestDeleteComplexExpr(t *testing.T) {
|
||||
expr string
|
||||
count int
|
||||
}
|
||||
capacity := common.TestCapacity
|
||||
exprLimits := []exprCount{
|
||||
{expr: fmt.Sprintf("%s >= 1000 || %s > 2000", common.DefaultInt64FieldName, common.DefaultInt64FieldName), count: 2000},
|
||||
|
||||
@ -463,7 +463,58 @@ func TestDeleteComplexExpr(t *testing.T) {
|
||||
|
||||
// data type not match and no error
|
||||
{expr: fmt.Sprintf("%s['number'] == '0' ", common.DefaultJSONFieldName), count: 0},
|
||||
}
|
||||
ch := make(chan struct{}, 5)
|
||||
wg := sync.WaitGroup{}
|
||||
testFunc := func(exprLimit exprCount) {
|
||||
defer func() {
|
||||
wg.Done()
|
||||
<-ch
|
||||
}()
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
// create collection and a partition
|
||||
cp := hp.NewCreateCollectionParams(hp.Int64VecAllScalar)
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, cp, hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true))
|
||||
|
||||
// insert [0, 3000) into default
|
||||
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption().TWithMaxCapacity(common.TestCapacity))
|
||||
prepare.FlushData(ctx, t, mc, schema.CollectionName)
|
||||
|
||||
// index and load
|
||||
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
|
||||
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
|
||||
|
||||
log.Debug("TestDeleteComplexExpr", zap.Any("expr", exprLimit.expr))
|
||||
|
||||
resDe, err := mc.Delete(ctx, client.NewDeleteOption(schema.CollectionName).WithExpr(exprLimit.expr))
|
||||
common.CheckErr(t, err, true)
|
||||
log.Debug("delete count", zap.Bool("equal", int64(exprLimit.count) == resDe.DeleteCount))
|
||||
// require.Equal(t, int64(exprLimit.count), resDe.DeleteCount)
|
||||
|
||||
resQuery, err := mc.Query(ctx, client.NewQueryOption(schema.CollectionName).WithFilter(exprLimit.expr).WithConsistencyLevel(entity.ClStrong))
|
||||
common.CheckErr(t, err, true)
|
||||
require.Zero(t, resQuery.ResultCount)
|
||||
}
|
||||
|
||||
for _, exprLimit := range exprLimits {
|
||||
exprLimit := exprLimit
|
||||
ch <- struct{}{}
|
||||
wg.Add(1)
|
||||
go testFunc(exprLimit)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// test delete json expr
|
||||
func TestDeleteComplexExprJson(t *testing.T) {
|
||||
type exprCount struct {
|
||||
expr string
|
||||
count int
|
||||
}
|
||||
capacity := common.TestCapacity
|
||||
exprLimits := []exprCount{
|
||||
// json field
|
||||
{expr: fmt.Sprintf("%s > 1499.5", common.DefaultJSONFieldName), count: 1500 / 2}, // json >= 1500.0
|
||||
{expr: fmt.Sprintf("%s like '21%%'", common.DefaultJSONFieldName), count: 100 / 4}, // json like '21%'
|
||||
|
||||
283
tests/go_client/testcases/full_text_search_test.go
Normal file
283
tests/go_client/testcases/full_text_search_test.go
Normal file
@ -0,0 +1,283 @@
|
||||
package testcases
|
||||
|
||||
import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/milvus-io/milvus/client/v2/entity"
|
||||
"github.com/milvus-io/milvus/client/v2/index"
|
||||
"github.com/milvus-io/milvus/client/v2/milvusclient"
|
||||
"github.com/milvus-io/milvus/tests/go_client/common"
|
||||
hp "github.com/milvus-io/milvus/tests/go_client/testcases/helper"
|
||||
)
|
||||
|
||||
func TestFullTextSearchDefault(t *testing.T) {
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
// create -> insert -> flush -> index -> load
|
||||
analyzerParams := map[string]any{"tokenizer": "standard"}
|
||||
fieldsOption := hp.TNewFieldsOption().TWithAnalyzerParams(analyzerParams)
|
||||
function := hp.TNewBM25Function(common.DefaultTextFieldName, common.DefaultTextSparseVecFieldName)
|
||||
schemaOption := hp.TNewSchemaOption().TWithFunction(function)
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.FullTextSearch), fieldsOption, schemaOption)
|
||||
insertOption := hp.TNewDataOption().TWithTextLang(common.DefaultTextLang)
|
||||
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), insertOption)
|
||||
prepare.FlushData(ctx, t, mc, schema.CollectionName)
|
||||
|
||||
indexparams := hp.TNewIndexParams(schema).TWithFieldIndex(map[string]index.Index{common.DefaultTextSparseVecFieldName: index.NewSparseInvertedIndex(entity.BM25, 0.1)})
|
||||
prepare.CreateIndex(ctx, t, mc, indexparams)
|
||||
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
|
||||
|
||||
// search
|
||||
queries := hp.GenFullTextQuery(common.DefaultNq, common.DefaultTextLang)
|
||||
vectors := make([]entity.Vector, 0, len(queries))
|
||||
for _, query := range queries {
|
||||
vectors = append(vectors, entity.Text(query))
|
||||
}
|
||||
resSearch, err := mc.Search(ctx, milvusclient.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).WithConsistencyLevel(entity.ClStrong))
|
||||
common.CheckErr(t, err, true)
|
||||
common.CheckSearchResult(t, resSearch, common.DefaultNq, common.DefaultLimit)
|
||||
}
|
||||
|
||||
// TestSearchFullTextBase tests basic full text search functionality with different languages
|
||||
func TestSearchFullTextWithDiffLang(t *testing.T) {
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
// Test cases for different languages and analyzers
|
||||
testCases := []struct {
|
||||
name string
|
||||
language string
|
||||
analyzer string
|
||||
query string
|
||||
numRows int
|
||||
topK int
|
||||
}{
|
||||
{
|
||||
name: "English_Standard",
|
||||
language: "english",
|
||||
analyzer: "standard",
|
||||
query: "what is information retrieval and its applications?",
|
||||
numRows: 3000,
|
||||
topK: 10,
|
||||
},
|
||||
{
|
||||
name: "Chinese_Jieba",
|
||||
language: "chinese",
|
||||
analyzer: "jieba",
|
||||
query: "信息检索的应用",
|
||||
numRows: 3000,
|
||||
topK: 10,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
analyzerParams := map[string]any{"tokenizer": tc.analyzer}
|
||||
fieldsOption := hp.TNewFieldsOption().TWithAnalyzerParams(analyzerParams)
|
||||
function := hp.TNewBM25Function(common.DefaultTextFieldName, common.DefaultTextSparseVecFieldName)
|
||||
schemaOption := hp.TNewSchemaOption().TWithFunction(function)
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.FullTextSearch), fieldsOption, schemaOption)
|
||||
insertOption := hp.TNewDataOption().TWithTextLang(tc.language).TWithNb(tc.numRows)
|
||||
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), insertOption)
|
||||
prepare.FlushData(ctx, t, mc, schema.CollectionName)
|
||||
|
||||
indexparams := hp.TNewIndexParams(schema).TWithFieldIndex(map[string]index.Index{common.DefaultTextSparseVecFieldName: index.NewSparseInvertedIndex(entity.BM25, 0.1)})
|
||||
prepare.CreateIndex(ctx, t, mc, indexparams)
|
||||
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
|
||||
|
||||
// search
|
||||
queries := []string{tc.query}
|
||||
vectors := make([]entity.Vector, 0, len(queries))
|
||||
for _, query := range queries {
|
||||
vectors = append(vectors, entity.Text(query))
|
||||
}
|
||||
resSearch, err := mc.Search(ctx, milvusclient.NewSearchOption(schema.CollectionName, tc.topK, vectors).WithConsistencyLevel(entity.ClStrong))
|
||||
common.CheckErr(t, err, true)
|
||||
common.CheckSearchResult(t, resSearch, len(queries), tc.topK)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSearchFullTextWithDynamicField tests full text search with dynamic field enabled
|
||||
func TestSearchFullTextWithDynamicField(t *testing.T) {
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
// Test cases for different languages and analyzers
|
||||
testCases := []struct {
|
||||
name string
|
||||
language string
|
||||
analyzer string
|
||||
query string
|
||||
numRows int
|
||||
topK int
|
||||
}{
|
||||
{
|
||||
name: "English_Standard",
|
||||
language: "english",
|
||||
analyzer: "standard",
|
||||
query: "what is information retrieval and its applications?",
|
||||
numRows: 1000,
|
||||
topK: 5,
|
||||
},
|
||||
{
|
||||
name: "Chinese_Jieba",
|
||||
language: "chinese",
|
||||
analyzer: "jieba",
|
||||
query: "信息检索的应用",
|
||||
numRows: 1000,
|
||||
topK: 5,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
analyzerParams := map[string]any{"tokenizer": tc.analyzer}
|
||||
fieldsOption := hp.TNewFieldsOption().TWithAnalyzerParams(analyzerParams)
|
||||
function := hp.TNewBM25Function(common.DefaultTextFieldName, common.DefaultTextSparseVecFieldName)
|
||||
schemaOption := hp.TNewSchemaOption().TWithFunction(function).TWithEnableDynamicField(true)
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.FullTextSearch), fieldsOption, schemaOption)
|
||||
insertOption := hp.TNewDataOption().TWithTextLang(tc.language).TWithNb(tc.numRows)
|
||||
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), insertOption)
|
||||
prepare.FlushData(ctx, t, mc, schema.CollectionName)
|
||||
|
||||
indexparams := hp.TNewIndexParams(schema).TWithFieldIndex(map[string]index.Index{common.DefaultTextSparseVecFieldName: index.NewSparseInvertedIndex(entity.BM25, 0.1)})
|
||||
prepare.CreateIndex(ctx, t, mc, indexparams)
|
||||
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
|
||||
|
||||
// search
|
||||
queries := []string{tc.query}
|
||||
vectors := make([]entity.Vector, 0, len(queries))
|
||||
for _, query := range queries {
|
||||
vectors = append(vectors, entity.Text(query))
|
||||
}
|
||||
resSearch, err := mc.Search(ctx, milvusclient.NewSearchOption(schema.CollectionName, tc.topK, vectors).WithConsistencyLevel(entity.ClStrong))
|
||||
common.CheckErr(t, err, true)
|
||||
common.CheckSearchResult(t, resSearch, len(queries), tc.topK)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSearchFullTextWithPartitionKey tests full text search with partition key
|
||||
func TestSearchFullTextWithPartitionKey(t *testing.T) {
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
// Test cases for different languages and analyzers
|
||||
testCases := []struct {
|
||||
name string
|
||||
language string
|
||||
analyzer string
|
||||
query string
|
||||
numRows int
|
||||
topK int
|
||||
}{
|
||||
{
|
||||
name: "English_Standard",
|
||||
language: "english",
|
||||
analyzer: "standard",
|
||||
query: "what is information retrieval and its applications?",
|
||||
numRows: 1000,
|
||||
topK: 5,
|
||||
},
|
||||
{
|
||||
name: "Chinese_Jieba",
|
||||
language: "chinese",
|
||||
analyzer: "jieba",
|
||||
query: "信息检索的应用",
|
||||
numRows: 1000,
|
||||
topK: 5,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
analyzerParams := map[string]any{"tokenizer": tc.analyzer}
|
||||
fieldsOption := hp.TNewFieldsOption().TWithAnalyzerParams(analyzerParams).TWithIsPartitionKey(true)
|
||||
function := hp.TNewBM25Function(common.DefaultTextFieldName, common.DefaultTextSparseVecFieldName)
|
||||
schemaOption := hp.TNewSchemaOption().TWithFunction(function)
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.FullTextSearch), fieldsOption, schemaOption)
|
||||
insertOption := hp.TNewDataOption().TWithTextLang(tc.language).TWithNb(tc.numRows)
|
||||
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), insertOption)
|
||||
prepare.FlushData(ctx, t, mc, schema.CollectionName)
|
||||
|
||||
indexparams := hp.TNewIndexParams(schema).TWithFieldIndex(map[string]index.Index{common.DefaultTextSparseVecFieldName: index.NewSparseInvertedIndex(entity.BM25, 0.1)})
|
||||
prepare.CreateIndex(ctx, t, mc, indexparams)
|
||||
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
|
||||
|
||||
// search
|
||||
queries := []string{tc.query}
|
||||
vectors := make([]entity.Vector, 0, len(queries))
|
||||
for _, query := range queries {
|
||||
vectors = append(vectors, entity.Text(query))
|
||||
}
|
||||
resSearch, err := mc.Search(ctx, milvusclient.NewSearchOption(schema.CollectionName, tc.topK, vectors).WithConsistencyLevel(entity.ClStrong))
|
||||
common.CheckErr(t, err, true)
|
||||
common.CheckSearchResult(t, resSearch, len(queries), tc.topK)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// TestSearchFullTextWithEmptyData tests full text search with empty data
|
||||
func TestSearchFullTextWithEmptyData(t *testing.T) {
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
// Test cases for different empty percent
|
||||
testCases := []struct {
|
||||
name string
|
||||
language string
|
||||
analyzer string
|
||||
query string
|
||||
numRows int
|
||||
topK int
|
||||
emptyPercent int
|
||||
}{
|
||||
{
|
||||
name: "English_Standard",
|
||||
language: "english",
|
||||
analyzer: "standard",
|
||||
query: "what is information retrieval and its applications?",
|
||||
numRows: 3000,
|
||||
topK: 5,
|
||||
emptyPercent: 50,
|
||||
},
|
||||
{
|
||||
name: "Chinese_Jieba",
|
||||
language: "chinese",
|
||||
analyzer: "jieba",
|
||||
query: "信息检索的应用",
|
||||
numRows: 3000,
|
||||
topK: 5,
|
||||
emptyPercent: 80,
|
||||
},
|
||||
}
|
||||
|
||||
for _, tc := range testCases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
analyzerParams := map[string]any{"tokenizer": tc.analyzer}
|
||||
fieldsOption := hp.TNewFieldsOption().TWithAnalyzerParams(analyzerParams).TWithIsPartitionKey(true)
|
||||
function := hp.TNewBM25Function(common.DefaultTextFieldName, common.DefaultTextSparseVecFieldName)
|
||||
schemaOption := hp.TNewSchemaOption().TWithFunction(function)
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.FullTextSearch), fieldsOption, schemaOption)
|
||||
insertOption := hp.TNewDataOption().TWithTextLang(tc.language).TWithNb(tc.numRows).TWithTextEmptyPercent(tc.emptyPercent)
|
||||
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), insertOption)
|
||||
prepare.FlushData(ctx, t, mc, schema.CollectionName)
|
||||
|
||||
indexparams := hp.TNewIndexParams(schema).TWithFieldIndex(map[string]index.Index{common.DefaultTextSparseVecFieldName: index.NewSparseInvertedIndex(entity.BM25, 0.1)})
|
||||
prepare.CreateIndex(ctx, t, mc, indexparams)
|
||||
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
|
||||
|
||||
// search
|
||||
queries := []string{tc.query}
|
||||
vectors := make([]entity.Vector, 0, len(queries))
|
||||
for _, query := range queries {
|
||||
vectors = append(vectors, entity.Text(query))
|
||||
}
|
||||
resSearch, err := mc.Search(ctx, milvusclient.NewSearchOption(schema.CollectionName, tc.topK, vectors).WithConsistencyLevel(entity.ClStrong))
|
||||
common.CheckErr(t, err, true)
|
||||
common.CheckSearchResult(t, resSearch, len(queries), tc.topK)
|
||||
})
|
||||
}
|
||||
}
|
||||
@ -4,6 +4,7 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -91,7 +92,15 @@ func prepareDataForGroupBySearch(t *testing.T, loopInsert int, insertNi int, idx
|
||||
// output_fields: pk + groupBy
|
||||
func TestSearchGroupByFloatDefault(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, idx := range genGroupByVectorIndex(entity.L2) {
|
||||
concurrency := 10
|
||||
ch := make(chan struct{}, concurrency)
|
||||
wg := sync.WaitGroup{}
|
||||
|
||||
testFunc := func(idx index.Index) {
|
||||
defer func() {
|
||||
wg.Done()
|
||||
<-ch
|
||||
}()
|
||||
// prepare data
|
||||
mc, ctx, collName := prepareDataForGroupBySearch(t, 100, 200, idx, false)
|
||||
|
||||
@ -144,11 +153,25 @@ func TestSearchGroupByFloatDefault(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, idx := range genGroupByVectorIndex(entity.L2) {
|
||||
ch <- struct{}{}
|
||||
wg.Add(1)
|
||||
go testFunc(idx)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func TestSearchGroupByFloatDefaultCosine(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, idx := range genGroupByVectorIndex(entity.COSINE) {
|
||||
concurrency := 10
|
||||
ch := make(chan struct{}, concurrency)
|
||||
wg := sync.WaitGroup{}
|
||||
|
||||
testFunc := func(idx index.Index) {
|
||||
defer func() {
|
||||
wg.Done()
|
||||
<-ch
|
||||
}()
|
||||
// prepare data
|
||||
mc, ctx, collName := prepareDataForGroupBySearch(t, 100, 200, idx, false)
|
||||
|
||||
@ -196,6 +219,12 @@ func TestSearchGroupByFloatDefaultCosine(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
for _, idx := range genGroupByVectorIndex(entity.COSINE) {
|
||||
ch <- struct{}{}
|
||||
wg.Add(1)
|
||||
go testFunc(idx)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// test groupBy search sparse vector
|
||||
@ -327,7 +356,16 @@ func TestSearchGroupByBinaryGrowing(t *testing.T) {
|
||||
// groupBy in growing segments, maybe growing index or brute force
|
||||
func TestSearchGroupByFloatGrowing(t *testing.T) {
|
||||
t.Parallel()
|
||||
for _, metricType := range hp.SupportFloatMetricType {
|
||||
concurrency := 10
|
||||
ch := make(chan struct{}, concurrency)
|
||||
wg := sync.WaitGroup{}
|
||||
|
||||
testFunc := func(metricType entity.MetricType) {
|
||||
defer func() {
|
||||
wg.Done()
|
||||
<-ch
|
||||
}()
|
||||
|
||||
idxHnsw := index.NewHNSWIndex(metricType, 8, 96)
|
||||
mc, ctx, collName := prepareDataForGroupBySearch(t, 100, 200, idxHnsw, true)
|
||||
|
||||
@ -376,6 +414,13 @@ func TestSearchGroupByFloatGrowing(t *testing.T) {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
for _, metricType := range hp.SupportFloatMetricType {
|
||||
ch <- struct{}{}
|
||||
wg.Add(1)
|
||||
go testFunc(metricType)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// groupBy + pagination
|
||||
|
||||
@ -1,5 +1,9 @@
|
||||
package helper
|
||||
|
||||
import (
|
||||
"github.com/milvus-io/milvus/client/v2/entity"
|
||||
)
|
||||
|
||||
type CreateCollectionParams struct {
|
||||
CollectionFieldsType CollectionFieldsType // collection fields type
|
||||
}
|
||||
@ -9,3 +13,25 @@ func NewCreateCollectionParams(collectionFieldsType CollectionFieldsType) *Creat
|
||||
CollectionFieldsType: collectionFieldsType,
|
||||
}
|
||||
}
|
||||
|
||||
type CreateCollectionOpt func(opt *createCollectionOpt)
|
||||
|
||||
type createCollectionOpt struct {
|
||||
shardNum int32
|
||||
enabledDynamicSchema bool
|
||||
|
||||
consistencyLevel entity.ConsistencyLevel
|
||||
properties map[string]any
|
||||
}
|
||||
|
||||
func TWithShardNum(shardNum int32) CreateCollectionOpt {
|
||||
return func(opt *createCollectionOpt) {
|
||||
opt.shardNum = shardNum
|
||||
}
|
||||
}
|
||||
|
||||
func TWithProperties(properties map[string]any) CreateCollectionOpt {
|
||||
return func(opt *createCollectionOpt) {
|
||||
opt.properties = properties
|
||||
}
|
||||
}
|
||||
|
||||
@ -3,7 +3,11 @@ package helper
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
@ -38,14 +42,16 @@ func (opt *InsertParams) TWithIsRows(isRows bool) *InsertParams {
|
||||
|
||||
// GenColumnDataOption -- create column data --
|
||||
type GenDataOption struct {
|
||||
nb int
|
||||
start int
|
||||
dim int
|
||||
maxLen int
|
||||
sparseMaxLen int
|
||||
maxCapacity int
|
||||
elementType entity.FieldType
|
||||
fieldName string
|
||||
nb int
|
||||
start int
|
||||
dim int
|
||||
maxLen int
|
||||
sparseMaxLen int
|
||||
maxCapacity int
|
||||
elementType entity.FieldType
|
||||
fieldName string
|
||||
textLang string
|
||||
textEmptyPercent int
|
||||
}
|
||||
|
||||
func (opt *GenDataOption) TWithNb(nb int) *GenDataOption {
|
||||
@ -88,15 +94,28 @@ func (opt *GenDataOption) TWithElementType(eleType entity.FieldType) *GenDataOpt
|
||||
return opt
|
||||
}
|
||||
|
||||
func (opt *GenDataOption) TWithTextLang(lang string) *GenDataOption {
|
||||
opt.textLang = lang
|
||||
return opt
|
||||
}
|
||||
|
||||
func (opt *GenDataOption) TWithTextEmptyPercent(percent int) *GenDataOption {
|
||||
opt.textEmptyPercent = percent
|
||||
return opt
|
||||
}
|
||||
|
||||
func TNewDataOption() *GenDataOption {
|
||||
return &GenDataOption{
|
||||
nb: common.DefaultNb,
|
||||
start: 0,
|
||||
dim: common.DefaultDim,
|
||||
maxLen: common.TestMaxLen,
|
||||
sparseMaxLen: common.TestMaxLen,
|
||||
maxCapacity: common.TestCapacity,
|
||||
elementType: entity.FieldTypeNone,
|
||||
nb: common.DefaultNb,
|
||||
start: 0,
|
||||
dim: common.DefaultDim,
|
||||
maxLen: common.TestMaxLen,
|
||||
sparseMaxLen: common.TestMaxLen,
|
||||
maxCapacity: common.TestCapacity,
|
||||
elementType: entity.FieldTypeNone,
|
||||
fieldName: "",
|
||||
textLang: "",
|
||||
textEmptyPercent: 0,
|
||||
}
|
||||
}
|
||||
|
||||
@ -197,10 +216,13 @@ func GenArrayColumnData(nb int, eleType entity.FieldType, option GenDataOption)
|
||||
}
|
||||
|
||||
type JSONStruct struct {
|
||||
Number int32 `json:"number,omitempty" milvus:"name:number"`
|
||||
String string `json:"string,omitempty" milvus:"name:string"`
|
||||
Number int32 `json:"number,omitempty" milvus:"name:number"`
|
||||
String string `json:"string,omitempty" milvus:"name:string"`
|
||||
Float float32 `json:"float,omitempty" milvus:"name:float"`
|
||||
*BoolStruct
|
||||
List []int64 `json:"list,omitempty" milvus:"name:list"`
|
||||
List []int64 `json:"list,omitempty" milvus:"name:list"`
|
||||
FloatArray []float64 `json:"floatArray,omitempty" milvus:"name:floatArray"`
|
||||
StringArray []string `json:"stringArray,omitempty" milvus:"name:stringArray"`
|
||||
}
|
||||
|
||||
// GenDefaultJSONData gen default column with data
|
||||
@ -216,12 +238,15 @@ func GenDefaultJSONData(nb int, option GenDataOption) [][]byte {
|
||||
if i < (start+nb)/2 {
|
||||
if i%2 == 0 {
|
||||
m = JSONStruct{
|
||||
String: strconv.Itoa(i),
|
||||
BoolStruct: _bool,
|
||||
String: strconv.Itoa(i),
|
||||
BoolStruct: _bool,
|
||||
FloatArray: []float64{float64(i), float64(i), float64(i)},
|
||||
StringArray: []string{fmt.Sprintf("%05d", i)},
|
||||
}
|
||||
} else {
|
||||
m = JSONStruct{
|
||||
Number: int32(i),
|
||||
Float: float32(i),
|
||||
String: strconv.Itoa(i),
|
||||
BoolStruct: _bool,
|
||||
List: []int64{int64(i), int64(i + 1)},
|
||||
@ -249,6 +274,24 @@ func GenDefaultJSONData(nb int, option GenDataOption) [][]byte {
|
||||
return jsonValues
|
||||
}
|
||||
|
||||
func GenNestedJSON(depth int, value any) map[string]interface{} {
|
||||
if depth == 1 {
|
||||
return map[string]interface{}{"value": value}
|
||||
}
|
||||
return map[string]interface{}{
|
||||
fmt.Sprintf("level%d", depth): GenNestedJSON(depth-1, value),
|
||||
}
|
||||
}
|
||||
|
||||
func GenNestedJSONExprKey(depth int, jsonField string) string {
|
||||
var pathParts []string
|
||||
for i := depth; i > 1; i-- {
|
||||
pathParts = append(pathParts, fmt.Sprintf("level%d", i))
|
||||
}
|
||||
pathParts = append(pathParts, "value")
|
||||
return fmt.Sprintf("%s['%s']", jsonField, strings.Join(pathParts, "']['"))
|
||||
}
|
||||
|
||||
// GenColumnData GenColumnDataOption except dynamic column
|
||||
func GenColumnData(nb int, fieldType entity.FieldType, option GenDataOption) column.Column {
|
||||
dim := option.dim
|
||||
@ -310,8 +353,35 @@ func GenColumnData(nb int, fieldType entity.FieldType, option GenDataOption) col
|
||||
|
||||
case entity.FieldTypeVarChar:
|
||||
varcharValues := make([]string, 0, nb)
|
||||
for i := start; i < start+nb; i++ {
|
||||
varcharValues = append(varcharValues, strconv.Itoa(i))
|
||||
if option.textLang != "" {
|
||||
// Use language-specific text generation
|
||||
var lang string
|
||||
switch option.textLang {
|
||||
case "en", "english":
|
||||
lang = "en"
|
||||
case "zh", "chinese":
|
||||
lang = "zh"
|
||||
default:
|
||||
// Fallback to sequential numbers for unsupported languages
|
||||
for i := start; i < start+nb; i++ {
|
||||
varcharValues = append(varcharValues, strconv.Itoa(i))
|
||||
}
|
||||
return column.NewColumnVarChar(fieldName, varcharValues)
|
||||
}
|
||||
|
||||
// Generate text data with empty values based on textEmptyPercent
|
||||
for i := 0; i < nb; i++ {
|
||||
if rand.Float64()*100 < float64(option.textEmptyPercent) {
|
||||
varcharValues = append(varcharValues, "")
|
||||
} else {
|
||||
varcharValues = append(varcharValues, common.GenText(lang))
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// Default behavior: sequential numbers
|
||||
for i := start; i < start+nb; i++ {
|
||||
varcharValues = append(varcharValues, strconv.Itoa(i))
|
||||
}
|
||||
}
|
||||
return column.NewColumnVarChar(fieldName, varcharValues)
|
||||
|
||||
@ -449,6 +519,16 @@ func MergeColumnsToDynamic(nb int, columns []column.Column, columnName string) *
|
||||
return jsonColumn
|
||||
}
|
||||
|
||||
func GetBm25FunctionsOutputFields(schema *entity.Schema) []string {
|
||||
var outputFields []string
|
||||
for _, fn := range schema.Functions {
|
||||
if fn.Type == entity.FunctionTypeBM25 {
|
||||
outputFields = append(outputFields, fn.OutputFieldNames...)
|
||||
}
|
||||
}
|
||||
return outputFields
|
||||
}
|
||||
|
||||
func GenColumnsBasedSchema(schema *entity.Schema, option *GenDataOption) ([]column.Column, []column.Column) {
|
||||
if nil == schema || schema.CollectionName == "" {
|
||||
log.Fatal("[GenColumnsBasedSchema] Nil Schema is not expected")
|
||||
@ -463,6 +543,16 @@ func GenColumnsBasedSchema(schema *entity.Schema, option *GenDataOption) ([]colu
|
||||
if field.AutoID {
|
||||
continue
|
||||
}
|
||||
option.fieldName = field.Name
|
||||
if option.fieldName == "" {
|
||||
option.fieldName = field.Name
|
||||
}
|
||||
if slices.Contains(GetBm25FunctionsOutputFields(schema), field.Name) {
|
||||
continue
|
||||
}
|
||||
log.Info("GenColumnsBasedSchema", zap.Any("field", field))
|
||||
// set field name to option
|
||||
option.TWithFieldName(field.Name)
|
||||
columns = append(columns, GenColumnData(option.nb, field.DataType, *option))
|
||||
}
|
||||
if schema.EnableDynamicField {
|
||||
|
||||
@ -107,6 +107,7 @@ const (
|
||||
Int64MultiVec CollectionFieldsType = 6 // int64 + floatVec + binaryVec + fp16Vec + bf16vec
|
||||
AllFields CollectionFieldsType = 7 // all fields excepted sparse
|
||||
Int64VecAllScalar CollectionFieldsType = 8 // int64 + floatVec + all scalar fields
|
||||
FullTextSearch CollectionFieldsType = 9 // int64 + varchar + sparse vector + analyzer + function
|
||||
)
|
||||
|
||||
type GenFieldsOption struct {
|
||||
@ -116,6 +117,8 @@ type GenFieldsOption struct {
|
||||
MaxLength int64 // varchar len or array capacity
|
||||
MaxCapacity int64
|
||||
IsPartitionKey bool
|
||||
EnableAnalyzer bool
|
||||
AnalyzerParams map[string]any
|
||||
ElementType entity.FieldType
|
||||
}
|
||||
|
||||
@ -127,6 +130,8 @@ func TNewFieldsOption() *GenFieldsOption {
|
||||
MaxCapacity: common.TestCapacity,
|
||||
IsDynamic: false,
|
||||
IsPartitionKey: false,
|
||||
EnableAnalyzer: false,
|
||||
AnalyzerParams: make(map[string]any),
|
||||
ElementType: entity.FieldTypeNone,
|
||||
}
|
||||
}
|
||||
@ -166,6 +171,16 @@ func (opt *GenFieldsOption) TWithMaxCapacity(maxCapacity int64) *GenFieldsOption
|
||||
return opt
|
||||
}
|
||||
|
||||
func (opt *GenFieldsOption) TWithEnableAnalyzer(enableAnalyzer bool) *GenFieldsOption {
|
||||
opt.EnableAnalyzer = enableAnalyzer
|
||||
return opt
|
||||
}
|
||||
|
||||
func (opt *GenFieldsOption) TWithAnalyzerParams(analyzerParams map[string]any) *GenFieldsOption {
|
||||
opt.AnalyzerParams = analyzerParams
|
||||
return opt
|
||||
}
|
||||
|
||||
// factory
|
||||
type FieldsFactory struct{}
|
||||
|
||||
@ -341,6 +356,23 @@ func (cf FieldsInt64VecAllScalar) GenFields(option GenFieldsOption) []*entity.Fi
|
||||
return fields
|
||||
}
|
||||
|
||||
type FieldsFullTextSearch struct{}
|
||||
|
||||
func (cf FieldsFullTextSearch) GenFields(option GenFieldsOption) []*entity.Field {
|
||||
pkField := entity.NewField().WithName(GetFieldNameByFieldType(entity.FieldTypeInt64)).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)
|
||||
textField := entity.NewField().WithName(common.DefaultTextFieldName).WithDataType(entity.FieldTypeVarChar).WithMaxLength(option.MaxLength).WithIsPartitionKey(option.IsPartitionKey).WithEnableAnalyzer(true).WithAnalyzerParams(option.AnalyzerParams)
|
||||
sparseVecField := entity.NewField().WithName(common.DefaultTextSparseVecFieldName).WithDataType(entity.FieldTypeSparseVector)
|
||||
if option.AutoID {
|
||||
pkField.WithIsAutoID(option.AutoID)
|
||||
}
|
||||
fields := []*entity.Field{
|
||||
pkField,
|
||||
textField,
|
||||
sparseVecField,
|
||||
}
|
||||
return fields
|
||||
}
|
||||
|
||||
func (ff FieldsFactory) GenFieldsForCollection(collectionFieldsType CollectionFieldsType, option *GenFieldsOption) []*entity.Field {
|
||||
log.Info("GenFieldsForCollection", zap.Any("GenFieldsOption", option))
|
||||
switch collectionFieldsType {
|
||||
@ -360,6 +392,8 @@ func (ff FieldsFactory) GenFieldsForCollection(collectionFieldsType CollectionFi
|
||||
return FieldsAllFields{}.GenFields(*option)
|
||||
case Int64VecAllScalar:
|
||||
return FieldsInt64VecAllScalar{}.GenFields(*option)
|
||||
case FullTextSearch:
|
||||
return FieldsFullTextSearch{}.GenFields(*option)
|
||||
default:
|
||||
return FieldsInt64Vec{}.GenFields(*option)
|
||||
}
|
||||
|
||||
14
tests/go_client/testcases/helper/function_helper.go
Normal file
14
tests/go_client/testcases/helper/function_helper.go
Normal file
@ -0,0 +1,14 @@
|
||||
package helper
|
||||
|
||||
import (
|
||||
"github.com/milvus-io/milvus/client/v2/entity"
|
||||
)
|
||||
|
||||
// TNewBM25Function creates a new BM25 function with the given input and output fields
|
||||
func TNewBM25Function(inputField, outputField string) *entity.Function {
|
||||
return entity.NewFunction().
|
||||
WithName(inputField + "_bm25_emb").
|
||||
WithInputFields(inputField).
|
||||
WithOutputFields(outputField).
|
||||
WithType(entity.FunctionTypeBM25)
|
||||
}
|
||||
@ -104,7 +104,7 @@ func GetInvalidPartitionKeyFieldType() []entity.FieldType {
|
||||
return nonPkFieldTypes
|
||||
}
|
||||
|
||||
// ----------------- prepare data --------------------------
|
||||
// CollectionPrepare ----------------- prepare data --------------------------
|
||||
type CollectionPrepare struct{}
|
||||
|
||||
var (
|
||||
@ -112,14 +112,44 @@ var (
|
||||
FieldsFact FieldsFactory
|
||||
)
|
||||
|
||||
func mergeOptions(schema *entity.Schema, opts ...CreateCollectionOpt) clientv2.CreateCollectionOption {
|
||||
//
|
||||
collectionOption := clientv2.NewCreateCollectionOption(schema.CollectionName, schema)
|
||||
tmpOption := &createCollectionOpt{}
|
||||
for _, o := range opts {
|
||||
o(tmpOption)
|
||||
}
|
||||
|
||||
if !common.IsZeroValue(tmpOption.shardNum) {
|
||||
collectionOption.WithShardNum(tmpOption.shardNum)
|
||||
}
|
||||
|
||||
if !common.IsZeroValue(tmpOption.enabledDynamicSchema) {
|
||||
collectionOption.WithDynamicSchema(tmpOption.enabledDynamicSchema)
|
||||
}
|
||||
|
||||
if !common.IsZeroValue(tmpOption.properties) {
|
||||
for k, v := range tmpOption.properties {
|
||||
collectionOption.WithProperty(k, v)
|
||||
}
|
||||
}
|
||||
|
||||
if !common.IsZeroValue(tmpOption.consistencyLevel) {
|
||||
collectionOption.WithConsistencyLevel(tmpOption.consistencyLevel)
|
||||
}
|
||||
|
||||
return collectionOption
|
||||
}
|
||||
|
||||
func (chainTask *CollectionPrepare) CreateCollection(ctx context.Context, t *testing.T, mc *base.MilvusClient,
|
||||
cp *CreateCollectionParams, fieldOpt *GenFieldsOption, schemaOpt *GenSchemaOption,
|
||||
cp *CreateCollectionParams, fieldOpt *GenFieldsOption, schemaOpt *GenSchemaOption, opts ...CreateCollectionOpt,
|
||||
) (*CollectionPrepare, *entity.Schema) {
|
||||
fields := FieldsFact.GenFieldsForCollection(cp.CollectionFieldsType, fieldOpt)
|
||||
schemaOpt.Fields = fields
|
||||
schema := GenSchema(schemaOpt)
|
||||
|
||||
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(schema.CollectionName, schema))
|
||||
createCollectionOption := mergeOptions(schema, opts...)
|
||||
err := mc.CreateCollection(ctx, createCollectionOption)
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
t.Cleanup(func() {
|
||||
@ -141,6 +171,8 @@ func (chainTask *CollectionPrepare) InsertData(ctx context.Context, t *testing.T
|
||||
if nil == ip.Schema || ip.Schema.CollectionName == "" {
|
||||
log.Fatal("[InsertData] Nil Schema is not expected")
|
||||
}
|
||||
// print option
|
||||
log.Info("GenDataOption", zap.Any("option", option))
|
||||
columns, dynamicColumns := GenColumnsBasedSchema(ip.Schema, option)
|
||||
insertOpt := clientv2.NewColumnBasedInsertOption(ip.Schema.CollectionName).WithColumns(columns...).WithColumns(dynamicColumns...)
|
||||
if ip.PartitionName != "" {
|
||||
|
||||
@ -55,6 +55,10 @@ var SupportBinIvfFlatMetricType = []entity.MetricType{
|
||||
entity.HAMMING,
|
||||
}
|
||||
|
||||
var SupportFullTextSearchMetricsType = []entity.MetricType{
|
||||
entity.BM25,
|
||||
}
|
||||
|
||||
var UnsupportedSparseVecMetricsType = []entity.MetricType{
|
||||
entity.L2,
|
||||
entity.COSINE,
|
||||
|
||||
@ -66,6 +66,14 @@ func GenSearchVectors(nq int, dim int, dataType entity.FieldType) []entity.Vecto
|
||||
return vectors
|
||||
}
|
||||
|
||||
func GenFullTextQuery(nq int, lang string) []string {
|
||||
queries := make([]string, 0, nq)
|
||||
for i := 0; i < nq; i++ {
|
||||
queries = append(queries, common.GenText(lang))
|
||||
}
|
||||
return queries
|
||||
}
|
||||
|
||||
func GenFp16OrBf16VectorsFromFloatVector(nq int, dim int, dataType entity.FieldType) []entity.Vector {
|
||||
vectors := make([]entity.Vector, 0, nq)
|
||||
switch dataType {
|
||||
|
||||
@ -12,6 +12,7 @@ type GenSchemaOption struct {
|
||||
AutoID bool
|
||||
Fields []*entity.Field
|
||||
EnableDynamicField bool
|
||||
Function *entity.Function
|
||||
}
|
||||
|
||||
func TNewSchemaOption() *GenSchemaOption {
|
||||
@ -43,6 +44,11 @@ func (opt *GenSchemaOption) TWithFields(fields []*entity.Field) *GenSchemaOption
|
||||
return opt
|
||||
}
|
||||
|
||||
func (opt *GenSchemaOption) TWithFunction(function *entity.Function) *GenSchemaOption {
|
||||
opt.Function = function
|
||||
return opt
|
||||
}
|
||||
|
||||
func GenSchema(option *GenSchemaOption) *entity.Schema {
|
||||
if len(option.Fields) == 0 {
|
||||
log.Fatal("Require at least a primary field and a vector field")
|
||||
@ -64,5 +70,8 @@ func GenSchema(option *GenSchemaOption) *entity.Schema {
|
||||
if option.EnableDynamicField {
|
||||
schema.WithDynamicFieldEnabled(option.EnableDynamicField)
|
||||
}
|
||||
if option.Function != nil {
|
||||
schema.WithFunction(option.Function)
|
||||
}
|
||||
return schema
|
||||
}
|
||||
|
||||
429
tests/go_client/testcases/hybrid_search_test.go
Normal file
429
tests/go_client/testcases/hybrid_search_test.go
Normal file
@ -0,0 +1,429 @@
|
||||
package testcases
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/stretchr/testify/require"
|
||||
|
||||
"github.com/milvus-io/milvus/client/v2/column"
|
||||
"github.com/milvus-io/milvus/client/v2/entity"
|
||||
"github.com/milvus-io/milvus/client/v2/index"
|
||||
client "github.com/milvus-io/milvus/client/v2/milvusclient"
|
||||
"github.com/milvus-io/milvus/tests/go_client/common"
|
||||
hp "github.com/milvus-io/milvus/tests/go_client/testcases/helper"
|
||||
)
|
||||
|
||||
func TestHybridSearchDefault(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
// create -> insert [0, 3000) -> flush -> index -> load
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
|
||||
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption())
|
||||
prepare.FlushData(ctx, t, mc, schema.CollectionName)
|
||||
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
|
||||
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
|
||||
|
||||
// hybrid search
|
||||
queryVec1 := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector)
|
||||
queryVec2 := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector)
|
||||
|
||||
annReq1 := client.NewAnnRequest(common.DefaultFloatVecFieldName, common.DefaultLimit, queryVec1...).WithSearchParam("ef", "100")
|
||||
annReq2 := client.NewAnnRequest(common.DefaultFloatVecFieldName, common.DefaultLimit, queryVec2...)
|
||||
|
||||
searchRes, errSearch := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReq1, annReq2).WithOutputFields("*"))
|
||||
common.CheckErr(t, errSearch, true)
|
||||
common.CheckSearchResult(t, searchRes, common.DefaultNq, common.DefaultLimit)
|
||||
common.CheckOutputFields(t, []string{common.DefaultInt64FieldName, common.DefaultFloatVecFieldName}, searchRes[0].Fields)
|
||||
|
||||
// ignore growing
|
||||
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption().TWithStart(common.DefaultNb).TWithNb(500))
|
||||
annReq1.WithIgnoreGrowing(true)
|
||||
annReq2.WithIgnoreGrowing(true)
|
||||
searchRes, errSearch = mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReq1, annReq2))
|
||||
common.CheckErr(t, errSearch, true)
|
||||
common.CheckSearchResult(t, searchRes, common.DefaultNq, common.DefaultLimit)
|
||||
for _, hits := range searchRes {
|
||||
for _, id := range hits.IDs.(*column.ColumnInt64).Data() {
|
||||
require.Less(t, id, int64(common.DefaultNb))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHybridSearchTemplateParam(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
// create -> insert [0, 3000) -> flush -> index -> load
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64MultiVec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
|
||||
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption())
|
||||
prepare.FlushData(ctx, t, mc, schema.CollectionName)
|
||||
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
|
||||
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
|
||||
|
||||
// hybrid search
|
||||
queryVec1 := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector)
|
||||
queryVec2 := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloat16Vector)
|
||||
|
||||
int64Value := 100
|
||||
annReq1 := client.NewAnnRequest(common.DefaultFloatVecFieldName, common.DefaultLimit, queryVec1...).
|
||||
WithFilter(fmt.Sprintf("%s > {int64Value}", common.DefaultInt64FieldName)).WithTemplateParam("int64Value", int64Value)
|
||||
annReq2 := client.NewAnnRequest(common.DefaultFloat16VecFieldName, common.DefaultLimit, queryVec2...).
|
||||
WithFilter(fmt.Sprintf("%s > {int64Value}", common.DefaultInt64FieldName)).WithTemplateParam("int64Value", 200)
|
||||
searchRes, errSearch := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReq1, annReq2))
|
||||
common.CheckErr(t, errSearch, true)
|
||||
common.CheckSearchResult(t, searchRes, common.DefaultNq, common.DefaultLimit)
|
||||
for _, hits := range searchRes {
|
||||
for _, id := range hits.IDs.(*column.ColumnInt64).Data() {
|
||||
require.Greater(t, id, int64(int64Value))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// hybrid search default -> verify success
|
||||
func TestHybridSearchMultiVectorsDefault(t *testing.T) {
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
for _, enableDynamic := range []bool{false, true} {
|
||||
// create -> insert [0, 3000) -> flush -> index -> load
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.AllFields),
|
||||
hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(enableDynamic))
|
||||
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption().TWithNb(common.DefaultNb*3))
|
||||
prepare.FlushData(ctx, t, mc, schema.CollectionName)
|
||||
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
|
||||
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
|
||||
|
||||
// hybrid search with different limit
|
||||
type limitGroup struct {
|
||||
subLimit1 int
|
||||
subLimit2 int
|
||||
limit int
|
||||
}
|
||||
// Duplicates when aggregating multiple subquery results
|
||||
limits := []limitGroup{
|
||||
{subLimit1: 10, subLimit2: 5, limit: 8}, // actual limit 8
|
||||
{subLimit1: 10, subLimit2: 5, limit: 15}, // actual limit [10, 15]
|
||||
{subLimit1: 10, subLimit2: 5, limit: 20}, // actual limit [10, 15]
|
||||
}
|
||||
|
||||
expr := fmt.Sprintf("%s > 5", common.DefaultInt64FieldName)
|
||||
var allFieldsName []string
|
||||
for _, field := range schema.Fields {
|
||||
allFieldsName = append(allFieldsName, field.Name)
|
||||
}
|
||||
if enableDynamic {
|
||||
allFieldsName = append(allFieldsName, common.DefaultDynamicFieldName)
|
||||
}
|
||||
|
||||
ch := make(chan struct{}, 3)
|
||||
wg := sync.WaitGroup{}
|
||||
testFunc := func(reranker client.Reranker) {
|
||||
defer func() {
|
||||
wg.Done()
|
||||
<-ch
|
||||
}()
|
||||
queryVec1 := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector)
|
||||
queryVec2 := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloat16Vector)
|
||||
|
||||
for _, limit := range limits {
|
||||
// hybrid search
|
||||
annReq1 := client.NewAnnRequest(common.DefaultFloatVecFieldName, limit.subLimit1, queryVec1...).WithFilter(expr)
|
||||
annReq2 := client.NewAnnRequest(common.DefaultFloat16VecFieldName, limit.subLimit2, queryVec2...).WithFilter(expr)
|
||||
|
||||
searchRes, errSearch := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, limit.limit, annReq1, annReq2).
|
||||
WithReranker(reranker).WithOutputFields("*"))
|
||||
common.CheckErr(t, errSearch, true)
|
||||
actualLimitRange := make([]int, 2)
|
||||
actualLimitRange[0] = min(max(limit.subLimit1, limit.subLimit2), limit.limit)
|
||||
actualLimitRange[1] = min(limit.subLimit1+limit.subLimit2, limit.limit)
|
||||
require.Len(t, searchRes, common.DefaultNq)
|
||||
common.CheckOutputFields(t, allFieldsName, searchRes[0].Fields)
|
||||
for _, res := range searchRes {
|
||||
require.GreaterOrEqual(t, res.ResultCount, actualLimitRange[0])
|
||||
require.LessOrEqual(t, res.ResultCount, actualLimitRange[1])
|
||||
require.GreaterOrEqual(t, searchRes[0].IDs.Len(), actualLimitRange[0])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// search with different reranker
|
||||
for _, reranker := range []client.Reranker{
|
||||
client.NewRRFReranker(),
|
||||
client.NewWeightedReranker([]float64{0.8, 0.2}),
|
||||
client.NewWeightedReranker([]float64{0.0, 0.2}),
|
||||
client.NewWeightedReranker([]float64{0.4, 1.0}),
|
||||
} {
|
||||
reranker := reranker
|
||||
ch <- struct{}{}
|
||||
wg.Add(1)
|
||||
go testFunc(reranker)
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
}
|
||||
|
||||
// invalid limit: 0, -1, max+1
|
||||
// invalid WeightedReranker params
|
||||
// invalid fieldName: not exist
|
||||
// invalid metric type: mismatch
|
||||
func TestHybridSearchInvalidParams(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
// create -> insert -> flush -> index -> load
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64MultiVec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
|
||||
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption())
|
||||
prepare.FlushData(ctx, t, mc, schema.CollectionName)
|
||||
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
|
||||
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
|
||||
|
||||
// hybrid search with invalid limit
|
||||
|
||||
queryVec1 := hp.GenSearchVectors(1, common.DefaultDim, entity.FieldTypeFloatVector)
|
||||
queryVec2 := hp.GenSearchVectors(1, common.DefaultDim, entity.FieldTypeBinaryVector)
|
||||
|
||||
annReq1 := client.NewAnnRequest(common.DefaultFloatVecFieldName, common.DefaultLimit, queryVec1...)
|
||||
annReq2 := client.NewAnnRequest(common.DefaultBinaryVecFieldName, common.DefaultLimit, queryVec2...)
|
||||
|
||||
for _, invalidLimit := range []int{-1, 0, common.MaxTopK + 1} {
|
||||
// hybrid search with invalid limit
|
||||
_, err := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, invalidLimit, annReq1))
|
||||
common.CheckErr(t, err, false, "should be greater than 0", "should be in range [1, 16384]")
|
||||
|
||||
// annRequest with invalid limit
|
||||
annReq2 := client.NewAnnRequest(common.DefaultFloatVecFieldName, invalidLimit, queryVec1...)
|
||||
_, err = mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, invalidLimit, annReq2))
|
||||
common.CheckErr(t, err, false, "should be greater than 0", "should be in range [1, 16384]")
|
||||
}
|
||||
|
||||
// hybrid search with invalid WeightedReranker params
|
||||
for _, invalidRanker := range []client.Reranker{
|
||||
client.NewWeightedReranker([]float64{-1, 0.2}),
|
||||
client.NewWeightedReranker([]float64{1.2, 0.2}),
|
||||
client.NewWeightedReranker([]float64{0.2}),
|
||||
client.NewWeightedReranker([]float64{0.2, 0.7, 0.5}),
|
||||
} {
|
||||
_, errReranker := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReq1, annReq2).WithReranker(invalidRanker))
|
||||
common.CheckErr(t, errReranker, false, "rank param weight should be in range [0, 1]",
|
||||
"the length of weights param mismatch with ann search requests")
|
||||
}
|
||||
|
||||
// invalid fieldName: not exist
|
||||
annReq3 := client.NewAnnRequest("a", common.DefaultLimit, queryVec1...)
|
||||
_, errField := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReq3))
|
||||
common.CheckErr(t, errField, false, "failed to get field schema by name: fieldName(a) not found")
|
||||
|
||||
// invalid metric type: mismatch
|
||||
annReq4 := client.NewAnnRequest(common.DefaultFloatVecFieldName, common.DefaultLimit, queryVec1...).WithSearchParam("metric_type", "L2")
|
||||
_, errMetric := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReq4))
|
||||
common.CheckErr(t, errMetric, false, "metric type not match: invalid parameter")
|
||||
}
|
||||
|
||||
// vector type mismatch: vectors: float32, queryVec: binary
|
||||
// vector dim mismatch
|
||||
func TestHybridSearchInvalidVectors(t *testing.T) {
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
|
||||
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption().TWithNb(500))
|
||||
prepare.FlushData(ctx, t, mc, schema.CollectionName)
|
||||
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
|
||||
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
|
||||
|
||||
// vector dim or type mismatch
|
||||
for _, invalidVec := range [][]entity.Vector{
|
||||
hp.GenSearchVectors(2, common.DefaultDim*2, entity.FieldTypeFloatVector), // vector dim mismatch
|
||||
hp.GenSearchVectors(1, common.DefaultDim, entity.FieldTypeFloat16Vector), // vector type mismatch
|
||||
} {
|
||||
annReq := client.NewAnnRequest(common.DefaultFloatVecFieldName, common.DefaultLimit, invalidVec...)
|
||||
_, err := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReq))
|
||||
common.CheckErr(t, err, false, "vector dimension mismatch", "vector type must be the same")
|
||||
}
|
||||
}
|
||||
|
||||
// hybrid search Pagination -> verify success
|
||||
func TestHybridSearchMultiVectorsPagination(t *testing.T) {
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64MultiVec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
|
||||
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
|
||||
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
|
||||
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption().TWithNb(common.DefaultNb*5))
|
||||
prepare.FlushData(ctx, t, mc, schema.CollectionName)
|
||||
|
||||
// hybrid search with different offset
|
||||
|
||||
queryVec1 := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector)
|
||||
queryVec2 := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloat16Vector)
|
||||
annReqDef := client.NewAnnRequest(common.DefaultFloatVecFieldName, common.DefaultLimit, queryVec1...)
|
||||
|
||||
// offset 0, -1 -> 0
|
||||
for _, offset := range []int{0, -1} {
|
||||
searchRes, err := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReqDef).WithOffset(offset))
|
||||
common.CheckErr(t, err, true)
|
||||
common.CheckSearchResult(t, searchRes, common.DefaultNq, common.DefaultLimit)
|
||||
}
|
||||
|
||||
// check for invalid offset externally, not internally
|
||||
annReqOffset := client.NewAnnRequest(common.DefaultFloat16VecFieldName, common.DefaultLimit, queryVec2...).WithOffset(common.MaxTopK + 1)
|
||||
res, errSearch := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReqDef, annReqOffset))
|
||||
common.CheckErr(t, errSearch, true)
|
||||
common.CheckSearchResult(t, res, common.DefaultNq, common.DefaultLimit)
|
||||
|
||||
_, errSearch = mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReqDef, annReqOffset).WithOffset(common.MaxTopK+1))
|
||||
common.CheckErr(t, errSearch, false, "should be gte than 0", "(offset+limit) should be in range [1, 16384]")
|
||||
|
||||
// search with different reranker and offset
|
||||
for _, reranker := range []client.Reranker{
|
||||
client.NewRRFReranker(),
|
||||
client.NewWeightedReranker([]float64{0.8, 0.2}),
|
||||
client.NewWeightedReranker([]float64{0.0, 0.2}),
|
||||
client.NewWeightedReranker([]float64{0.4, 1.0}),
|
||||
} {
|
||||
annReq1 := client.NewAnnRequest(common.DefaultFloatVecFieldName, common.DefaultLimit, queryVec1...)
|
||||
annReq2 := client.NewAnnRequest(common.DefaultFloat16VecFieldName, common.DefaultLimit, queryVec2...)
|
||||
// hybrid search
|
||||
searchRes, errSearch := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReq1, annReq2).WithReranker(reranker))
|
||||
common.CheckErr(t, errSearch, true)
|
||||
|
||||
offsetRes, errSearch := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, 5, annReq1, annReq2).WithReranker(reranker).WithOffset(5))
|
||||
common.CheckErr(t, errSearch, true)
|
||||
common.CheckSearchResult(t, searchRes, common.DefaultNq, common.DefaultLimit)
|
||||
common.CheckSearchResult(t, offsetRes, common.DefaultNq, 5)
|
||||
for i := 0; i < len(searchRes); i++ {
|
||||
require.Equal(t, searchRes[i].IDs.(*column.ColumnInt64).Data()[5:], offsetRes[i].IDs.(*column.ColumnInt64).Data())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// hybrid search Pagination -> verify success
|
||||
func TestHybridSearchMultiVectorsRangeSearch(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
// create -> insert [0, 3000) -> flush -> index -> load
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64MultiVec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
|
||||
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption().TWithNb(common.DefaultNb*3))
|
||||
prepare.FlushData(ctx, t, mc, schema.CollectionName)
|
||||
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
|
||||
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
|
||||
|
||||
// hybrid search
|
||||
expr := fmt.Sprintf("%s > 4", common.DefaultInt64FieldName)
|
||||
queryVec1 := hp.GenSearchVectors(1, common.DefaultDim, entity.FieldTypeFloatVector)
|
||||
queryVec2 := hp.GenSearchVectors(1, common.DefaultDim, entity.FieldTypeFloat16Vector)
|
||||
|
||||
// search with different reranker and offset
|
||||
for _, reranker := range []client.Reranker{
|
||||
client.NewRRFReranker(),
|
||||
client.NewWeightedReranker([]float64{0.8, 0.2}),
|
||||
client.NewWeightedReranker([]float64{0.5, 0.5}),
|
||||
} {
|
||||
annReq1 := client.NewAnnRequest(common.DefaultFloatVecFieldName, common.DefaultLimit*2, queryVec1...).WithSearchParam("radius", "20").WithOffset(1).WithFilter(expr)
|
||||
annReq2 := client.NewAnnRequest(common.DefaultFloat16VecFieldName, common.DefaultLimit, queryVec2...).WithSearchParam("range_filter", "0.01").WithFilter(expr)
|
||||
// hybrid search
|
||||
resRange, errSearch := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReq1, annReq2).WithReranker(reranker))
|
||||
common.CheckErr(t, errSearch, true)
|
||||
common.CheckSearchResult(t, resRange, 1, common.DefaultLimit)
|
||||
for _, res := range resRange {
|
||||
for _, score := range res.Scores {
|
||||
require.GreaterOrEqual(t, score, float32(0.01))
|
||||
require.LessOrEqual(t, score, float32(20))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHybridSearchSparseVector(t *testing.T) {
|
||||
t.Parallel()
|
||||
idxInverted := index.NewSparseInvertedIndex(entity.IP, 0.2)
|
||||
idxWand := index.NewSparseWANDIndex(entity.IP, 0.3)
|
||||
|
||||
for _, idx := range []index.Index{idxInverted, idxWand} {
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
// create -> insert [0, 3000) -> flush -> index -> load
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VarcharSparseVec), hp.TNewFieldsOption(),
|
||||
hp.TNewSchemaOption().TWithEnableDynamicField(true))
|
||||
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema).TWithFieldIndex(map[string]index.Index{common.DefaultSparseVecFieldName: idx}))
|
||||
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
|
||||
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption().TWithNb(common.DefaultNb*3))
|
||||
prepare.FlushData(ctx, t, mc, schema.CollectionName)
|
||||
|
||||
// search
|
||||
queryVec1 := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim*2, entity.FieldTypeSparseVector)
|
||||
queryVec2 := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeSparseVector)
|
||||
|
||||
expr := fmt.Sprintf("%s > 1", common.DefaultInt64FieldName)
|
||||
for _, reranker := range []client.Reranker{
|
||||
client.NewRRFReranker(),
|
||||
client.NewWeightedReranker([]float64{0.5, 0.6}),
|
||||
} {
|
||||
// hybrid search
|
||||
annReq1 := client.NewAnnRequest(common.DefaultSparseVecFieldName, common.DefaultLimit, queryVec1...).WithFilter(expr)
|
||||
annReq2 := client.NewAnnRequest(common.DefaultSparseVecFieldName, common.DefaultLimit, queryVec2...)
|
||||
|
||||
searchRes, errSearch := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReq1, annReq2).
|
||||
WithReranker(reranker).WithOutputFields("*"))
|
||||
common.CheckErr(t, errSearch, true)
|
||||
common.CheckSearchResult(t, searchRes, common.DefaultNq, common.DefaultLimit)
|
||||
common.CheckErr(t, errSearch, true)
|
||||
outputFields := []string{common.DefaultInt64FieldName, common.DefaultVarcharFieldName, common.DefaultSparseVecFieldName, common.DefaultDynamicFieldName}
|
||||
common.CheckOutputFields(t, outputFields, searchRes[0].Fields)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestHybridSearchGroupBy(t *testing.T) {
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
// create collection
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.AllFields), hp.TNewFieldsOption(), hp.TNewSchemaOption())
|
||||
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
|
||||
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
|
||||
|
||||
ch := make(chan struct{}, 5)
|
||||
wg := sync.WaitGroup{}
|
||||
|
||||
testFunc := func() {
|
||||
defer func() {
|
||||
wg.Done()
|
||||
<-ch
|
||||
}()
|
||||
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption().TWithNb(1000))
|
||||
}
|
||||
|
||||
for i := 0; i < 10; i++ {
|
||||
ch <- struct{}{}
|
||||
wg.Add(1)
|
||||
go testFunc()
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
prepare.FlushData(ctx, t, mc, schema.CollectionName)
|
||||
|
||||
// hybrid search with groupby field
|
||||
|
||||
queryVec1 := hp.GenSearchVectors(2, common.DefaultDim, entity.FieldTypeFloatVector)
|
||||
queryVec2 := hp.GenSearchVectors(2, common.DefaultDim, entity.FieldTypeBFloat16Vector)
|
||||
annReq1 := client.NewAnnRequest(common.DefaultFloatVecFieldName, common.DefaultLimit, queryVec1...).WithGroupByField(common.DefaultVarcharFieldName)
|
||||
annReq2 := client.NewAnnRequest(common.DefaultBFloat16VecFieldName, common.DefaultLimit, queryVec2...).WithGroupByField(common.DefaultInt32FieldName)
|
||||
|
||||
res, errSearch := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReq1, annReq2))
|
||||
common.CheckErr(t, errSearch, true)
|
||||
common.CheckSearchResult(t, res, 2, common.DefaultLimit)
|
||||
|
||||
// TODO hybrid search WithGroupSize, WithStrictGroupSize
|
||||
}
|
||||
@ -17,7 +17,6 @@ import (
|
||||
)
|
||||
|
||||
func TestIndexVectorDefault(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
@ -50,7 +49,6 @@ func TestIndexVectorDefault(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestIndexVectorIP(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
@ -84,7 +82,6 @@ func TestIndexVectorIP(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestIndexVectorCosine(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
@ -118,7 +115,6 @@ func TestIndexVectorCosine(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestIndexAutoFloatVector(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
@ -155,7 +151,6 @@ func TestIndexAutoFloatVector(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestIndexAutoBinaryVector(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
@ -196,7 +191,6 @@ func TestIndexAutoBinaryVector(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestIndexAutoSparseVector(t *testing.T) {
|
||||
t.Parallel()
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
@ -758,7 +752,6 @@ func TestCreateIndexDup(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCreateIndexSparseVectorGeneric(t *testing.T) {
|
||||
t.Parallel()
|
||||
idxInverted := index.NewGenericIndex(common.DefaultSparseVecFieldName, map[string]string{"drop_ratio_build": "0.2", index.MetricTypeKey: "IP", index.IndexTypeKey: "SPARSE_INVERTED_INDEX"})
|
||||
idxWand := index.NewGenericIndex(common.DefaultSparseVecFieldName, map[string]string{"drop_ratio_build": "0.3", index.MetricTypeKey: "IP", index.IndexTypeKey: "SPARSE_WAND"})
|
||||
|
||||
@ -787,7 +780,6 @@ func TestCreateIndexSparseVectorGeneric(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestCreateIndexSparseVector(t *testing.T) {
|
||||
t.Parallel()
|
||||
idxInverted1 := index.NewSparseInvertedIndex(entity.IP, 0.2)
|
||||
idxWand1 := index.NewSparseWANDIndex(entity.IP, 0.3)
|
||||
for _, idx := range []index.Index{idxInverted1, idxWand1} {
|
||||
|
||||
@ -2,6 +2,7 @@ package testcases
|
||||
|
||||
import (
|
||||
"math"
|
||||
"strconv"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -126,6 +127,11 @@ func TestInsertAllFieldsData(t *testing.T) {
|
||||
flushTak, _ := mc.Flush(ctx, client.NewFlushOption(schema.CollectionName))
|
||||
err := flushTak.Await(ctx)
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
// check collection stats
|
||||
stats, err := mc.GetCollectionStats(ctx, client.NewGetCollectionStatsOption(schema.CollectionName))
|
||||
common.CheckErr(t, err, true)
|
||||
require.Equal(t, map[string]string{common.RowCount: strconv.Itoa(common.DefaultNb)}, stats)
|
||||
}
|
||||
}
|
||||
|
||||
@ -587,6 +593,11 @@ func TestInsertDefaultRows(t *testing.T) {
|
||||
common.CheckErr(t, errFlush, true)
|
||||
errFlush = flushTask.Await(ctx)
|
||||
common.CheckErr(t, errFlush, true)
|
||||
|
||||
// check collection stats
|
||||
stats, err := mc.GetCollectionStats(ctx, client.NewGetCollectionStatsOption(schema.CollectionName))
|
||||
common.CheckErr(t, err, true)
|
||||
require.Equal(t, map[string]string{common.RowCount: strconv.Itoa(common.DefaultNb)}, stats)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@ -33,10 +33,10 @@ func teardown() {
|
||||
defer mc.Close(ctx)
|
||||
|
||||
// clear dbs
|
||||
dbs, _ := mc.ListDatabases(ctx, clientv2.NewListDatabaseOption())
|
||||
dbs, _ := mc.ListDatabase(ctx, clientv2.NewListDatabaseOption())
|
||||
for _, db := range dbs {
|
||||
if db != common.DefaultDb {
|
||||
_ = mc.UsingDatabase(ctx, clientv2.NewUseDatabaseOption(db))
|
||||
_ = mc.UseDatabase(ctx, clientv2.NewUseDatabaseOption(db))
|
||||
collections, _ := mc.ListCollections(ctx, clientv2.NewListCollectionOption())
|
||||
for _, coll := range collections {
|
||||
_ = mc.DropCollection(ctx, clientv2.NewDropCollectionOption(coll))
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
package testcases
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
@ -604,6 +605,13 @@ func TestQueryCountJsonDynamicExpr(t *testing.T) {
|
||||
{expr: fmt.Sprintf("%s == [1503, 1504]", common.DefaultJSONFieldName), count: 1}, // json == [1,2]
|
||||
{expr: fmt.Sprintf("%s[0] > 1", common.DefaultJSONFieldName), count: 1500 / 4}, // json[0] > 1
|
||||
{expr: fmt.Sprintf("%s[0][0] > 1", common.DefaultJSONFieldName), count: 0}, // json == [1,2]
|
||||
|
||||
// Key and value types do not match
|
||||
{expr: fmt.Sprintf("%s['float'] <= 3000", common.DefaultJSONFieldName), count: common.DefaultNb / 4},
|
||||
{expr: fmt.Sprintf("%s['float'] <= 3000.0", common.DefaultJSONFieldName), count: common.DefaultNb / 4},
|
||||
{expr: fmt.Sprintf("%s['string'] > 0", common.DefaultJSONFieldName), count: 0},
|
||||
{expr: fmt.Sprintf("%s['floatArray'][0] < 1000.0", common.DefaultJSONFieldName), count: 500},
|
||||
{expr: fmt.Sprintf("%s['stringArray'][0] == '00100'", common.DefaultJSONFieldName), count: 1},
|
||||
}
|
||||
|
||||
for _, _exprCount := range exprCounts {
|
||||
@ -614,6 +622,50 @@ func TestQueryCountJsonDynamicExpr(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueryNestedJsonExpr(t *testing.T) {
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VecJSON), hp.TNewFieldsOption(), hp.TNewSchemaOption())
|
||||
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
|
||||
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
|
||||
|
||||
pkColumn := hp.GenColumnData(common.DefaultNb, entity.FieldTypeInt64, *hp.TNewDataOption())
|
||||
vecColumn := hp.GenColumnData(common.DefaultNb, entity.FieldTypeFloatVector, *hp.TNewDataOption())
|
||||
jsonValues := make([][]byte, 0, common.DefaultNb)
|
||||
nestedDepth := 100
|
||||
for i := 0; i < common.DefaultNb; i++ {
|
||||
var m map[string]interface{}
|
||||
if i%2 == 0 {
|
||||
m = make(map[string]interface{})
|
||||
} else {
|
||||
m = hp.GenNestedJSON(nestedDepth, i)
|
||||
}
|
||||
bs, _ := json.Marshal(&m)
|
||||
jsonValues = append(jsonValues, bs)
|
||||
}
|
||||
jsonColumn := column.NewColumnJSONBytes(common.DefaultJSONFieldName, jsonValues)
|
||||
_, err := mc.Insert(ctx, client.NewColumnBasedInsertOption(schema.CollectionName, pkColumn, vecColumn, jsonColumn))
|
||||
common.CheckErr(t, err, true)
|
||||
|
||||
type exprCount struct {
|
||||
expr string
|
||||
count int64
|
||||
}
|
||||
exprKey := hp.GenNestedJSONExprKey(nestedDepth, common.DefaultJSONFieldName)
|
||||
nestedExpr := exprKey + " < 1000 "
|
||||
t.Log("https://github.com/milvus-io/milvus/issues/39822")
|
||||
exprCounts := []exprCount{
|
||||
//{expr: fmt.Sprintf("json_length(%s) == 0", common.DefaultJSONFieldName), count: common.DefaultNb / 2},
|
||||
{expr: nestedExpr, count: 500},
|
||||
}
|
||||
for _, _exprCount := range exprCounts {
|
||||
log.Info("TestQueryCountJsonDynamicExpr", zap.String("expr", _exprCount.expr))
|
||||
countRes, _ := mc.Query(ctx, client.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(_exprCount.expr).WithOutputFields(common.QueryCountFieldName))
|
||||
count, _ := countRes.Fields[0].GetAsInt64(0)
|
||||
require.Equal(t, _exprCount.count, count)
|
||||
}
|
||||
}
|
||||
|
||||
// test query with all kinds of array expr
|
||||
func TestQueryArrayFieldExpr(t *testing.T) {
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
|
||||
@ -4,6 +4,7 @@ import (
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"strconv"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
@ -570,32 +571,44 @@ func TestSearchInvalidScannReorderK(t *testing.T) {
|
||||
// test search with scann index params: with_raw_data and metrics_type [L2, IP, COSINE]
|
||||
func TestSearchScannAllMetricsWithRawData(t *testing.T) {
|
||||
t.Parallel()
|
||||
ch := make(chan struct{}, 3)
|
||||
wg := sync.WaitGroup{}
|
||||
testFunc := func(withRawData bool, metricType entity.MetricType) {
|
||||
defer func() {
|
||||
wg.Done()
|
||||
<-ch
|
||||
}()
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VecJSON),
|
||||
hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true))
|
||||
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption())
|
||||
prepare.FlushData(ctx, t, mc, schema.CollectionName)
|
||||
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema).TWithFieldIndex(map[string]index.Index{
|
||||
common.DefaultFloatVecFieldName: index.NewSCANNIndex(metricType, 16, withRawData),
|
||||
}))
|
||||
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
|
||||
|
||||
// search and output all fields
|
||||
vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector)
|
||||
resSearch, errSearch := mc.Search(ctx, client.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).
|
||||
WithConsistencyLevel(entity.ClStrong).WithOutputFields("*"))
|
||||
common.CheckErr(t, errSearch, true)
|
||||
common.CheckOutputFields(t, []string{
|
||||
common.DefaultInt64FieldName, common.DefaultJSONFieldName,
|
||||
common.DefaultFloatVecFieldName, common.DefaultDynamicFieldName,
|
||||
}, resSearch[0].Fields)
|
||||
common.CheckSearchResult(t, resSearch, common.DefaultNq, common.DefaultLimit)
|
||||
}
|
||||
for _, withRawData := range []bool{true, false} {
|
||||
for _, metricType := range []entity.MetricType{entity.L2, entity.IP, entity.COSINE} {
|
||||
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
|
||||
mc := createDefaultMilvusClient(ctx, t)
|
||||
|
||||
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VecJSON),
|
||||
hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true))
|
||||
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption())
|
||||
prepare.FlushData(ctx, t, mc, schema.CollectionName)
|
||||
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema).TWithFieldIndex(map[string]index.Index{
|
||||
common.DefaultFloatVecFieldName: index.NewSCANNIndex(metricType, 16, withRawData),
|
||||
}))
|
||||
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
|
||||
|
||||
// search and output all fields
|
||||
vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector)
|
||||
resSearch, errSearch := mc.Search(ctx, client.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).
|
||||
WithConsistencyLevel(entity.ClStrong).WithOutputFields("*"))
|
||||
common.CheckErr(t, errSearch, true)
|
||||
common.CheckOutputFields(t, []string{
|
||||
common.DefaultInt64FieldName, common.DefaultJSONFieldName,
|
||||
common.DefaultFloatVecFieldName, common.DefaultDynamicFieldName,
|
||||
}, resSearch[0].Fields)
|
||||
common.CheckSearchResult(t, resSearch, common.DefaultNq, common.DefaultLimit)
|
||||
ch <- struct{}{}
|
||||
wg.Add(1)
|
||||
go testFunc(withRawData, metricType)
|
||||
}
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// test search with valid expression
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user