test: [GoSDK][2.5] Sync gosdk test commits from master branch (#39838)

Cherry-pick from master
pr: #39484 #39570 #39537 #39824 #39579

---------

Signed-off-by: ThreadDao <yufen.zong@zilliz.com>
Signed-off-by: zhuwenxing <wenxing.zhu@zilliz.com>
Signed-off-by: zhuwenxing <wxzhuyeah@gmail.com>
Co-authored-by: zhuwenxing <wenxing.zhu@zilliz.com>
This commit is contained in:
ThreadDao 2025-02-18 11:11:08 +08:00 committed by GitHub
parent 7ea0af572a
commit 7fb69960d1
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
26 changed files with 2004 additions and 125 deletions

View File

@ -87,7 +87,7 @@ func (c *Client) DescribeDatabase(ctx context.Context, option DescribeDatabaseOp
return db, err
}
func (c *Client) AlterDatabaseProperies(ctx context.Context, option AlterDatabasePropertiesOption, callOptions ...grpc.CallOption) error {
func (c *Client) AlterDatabaseProperties(ctx context.Context, option AlterDatabasePropertiesOption, callOptions ...grpc.CallOption) error {
req := option.Request()
return c.callService(func(milvusService milvuspb.MilvusServiceClient) error {

View File

@ -153,7 +153,7 @@ func (s *DatabaseSuite) TestAlterDatabaseProperties() {
return merr.Success(), nil
}).Once()
err := s.client.AlterDatabaseProperies(ctx, NewAlterDatabasePropertiesOption(dbName).WithProperty(key, value))
err := s.client.AlterDatabaseProperties(ctx, NewAlterDatabasePropertiesOption(dbName).WithProperty(key, value))
s.NoError(err)
})
@ -161,7 +161,7 @@ func (s *DatabaseSuite) TestAlterDatabaseProperties() {
dbName := fmt.Sprintf("dt_%s", s.randString(6))
s.mock.EXPECT().AlterDatabase(mock.Anything, mock.Anything).Return(nil, merr.WrapErrServiceInternal("mocked")).Once()
err := s.client.AlterDatabaseProperies(ctx, NewAlterDatabasePropertiesOption(dbName).WithProperty("key", "value"))
err := s.client.AlterDatabaseProperties(ctx, NewAlterDatabasePropertiesOption(dbName).WithProperty("key", "value"))
s.Error(err)
})
}

View File

@ -1 +1,191 @@
## go_client
# Milvus Go Client Test Framework
## Overview
This is a comprehensive test framework for the Milvus Go Client, designed to validate various functionalities of the Milvus vector database client. The framework provides a structured approach to writing tests with reusable components and helper functions.
## Framework Architecture
### Directory Structure
```
/go_client/
├── testcases/ # Main test cases
│ ├── helper/ # Helper functions and utilities
│ │ ├── helper.go
│ │ ├── data_helper.go
│ │ └── collection_helper.go
│ ├── search_test.go # Search functionality tests
│ ├── index_test.go # Index management tests
│ └── ...
├── common/ # Common utilities and constants
└── base/ # Base infrastructure code
```
### Key Components
- **Collection Preparation**: Utilities for creating and managing collections
- **Data Generation**: Tools for generating test data
- **Helper Functions**: Common operations and validations
- **Test Cases**: Organized by functionality
## Writing Test Cases
### Basic Test Structure
```go
func TestYourFeature(t *testing.T) {
// 1. Setup context and client
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// 2. Prepare collection
prepare, schema := hp.CollPrepare.CreateCollection(
ctx, t, mc,
hp.NewCreateCollectionParams(hp.Int64Vec),
hp.TNewFieldsOption(),
hp.TNewSchemaOption(),
)
// 3. Insert test data
prepare.InsertData(ctx, t, mc,
hp.NewInsertParams(schema),
hp.TNewDataOption(),
)
// 4. Execute test operations
// ... your test logic here ...
// 5. Validate results
require.NoError(t, err)
require.Equal(t, expected, actual)
}
```
### Using Custom Parameters
1. **Collection Creation Parameters**
```go
fieldsOption := hp.TNewFieldsOption().
TWithEnableAnalyzer(true).
TWithAnalyzerParams(map[string]any{
"tokenizer": "standard",
})
schemaOption := hp.TNewSchemaOption().
TWithEnableDynamicField(true).
TWithDescription("Custom schema").
TWithAutoID(false)
```
2. **Data Insertion Options**
```go
insertOption := hp.TNewDataOption().
TWithNb(1000). // Number of records
TWithDim(128). // Vector dimension
TWithStart(100). // Starting ID
TWithMaxLen(256). // Maximum length
TWithTextLang("en") // Text language
```
3. **Index Parameters**
```go
indexParams := hp.TNewIndexParams(schema).
TWithFieldIndex(map[string]index.Index{
common.DefaultVectorFieldName: index.NewIVFSQIndex(
&index.IVFSQConfig{
MetricType: entity.L2,
NList: 128,
},
),
})
```
4. **Search Parameters**
```go
searchOpt := client.NewSearchOption(schema.CollectionName, 100, vectors).
WithOffset(0).
WithLimit(100).
WithConsistencyLevel(entity.ClStrong).
WithFilter("int64 >= 100").
WithOutputFields([]string{"*"}).
WithSearchParams(map[string]any{
"nprobe": 16,
"ef": 64,
})
```
## Adding New Parameters
1. **Define New Option Type**
```go
// In helper/data_helper.go
type YourNewOption struct {
newParam1 string
newParam2 int
}
```
2. **Add Constructor**
```go
func TNewYourOption() *YourNewOption {
return &YourNewOption{
newParam1: "default",
newParam2: 0,
}
}
```
3. **Add Parameter Methods**
```go
func (opt *YourNewOption) TWithNewParam1(value string) *YourNewOption {
opt.newParam1 = value
return opt
}
func (opt *YourNewOption) TWithNewParam2(value int) *YourNewOption {
opt.newParam2 = value
return opt
}
```
## Best Practices
1. **Test Organization**
- Group related tests in the same file
- Use clear and descriptive test names
- Add comments explaining test purpose
2. **Data Generation**
- Use helper functions for generating test data
- Ensure data is appropriate for the test case
- Clean up test data after use
3. **Error Handling**
- Use `common.CheckErr` for consistent error checking
- Test both success and failure scenarios
- Validate error messages when appropriate
4. **Performance Considerations**
- Use appropriate timeouts
- Clean up resources after tests
- Consider test execution time
## Running Tests
```bash
# Run all tests
go test ./testcases/...
# Run specific test
go test -run TestYourFeature ./testcases/
# Run with verbose output
go test -v ./testcases/...
```
## Contributing
1. Follow the existing code structure
2. Add comprehensive test cases
3. Document new parameters and options
4. Update this README for significant changes
5. Ensure code quality standards:
- Run `golangci-lint run` to check for style mistakes
- Use `gofmt -w your/code/path` to format your code before submitting
- CI will verify both golint and go format compliance

View File

@ -88,14 +88,14 @@ func (mc *MilvusClient) Close(ctx context.Context) error {
// -- database --
// UsingDatabase list all database in milvus cluster.
func (mc *MilvusClient) UsingDatabase(ctx context.Context, option client.UseDatabaseOption) error {
// UseDatabase list all database in milvus cluster.
func (mc *MilvusClient) UseDatabase(ctx context.Context, option client.UseDatabaseOption) error {
err := mc.mClient.UseDatabase(ctx, option)
return err
}
// ListDatabases list all database in milvus cluster.
func (mc *MilvusClient) ListDatabases(ctx context.Context, option client.ListDatabaseOption, callOptions ...grpc.CallOption) ([]string, error) {
// ListDatabase list all database in milvus cluster.
func (mc *MilvusClient) ListDatabase(ctx context.Context, option client.ListDatabaseOption, callOptions ...grpc.CallOption) ([]string, error) {
databaseNames, err := mc.mClient.ListDatabase(ctx, option, callOptions...)
return databaseNames, err
}
@ -112,6 +112,24 @@ func (mc *MilvusClient) DropDatabase(ctx context.Context, option client.DropData
return err
}
// DescribeDatabase describe database with the given db name.
func (mc *MilvusClient) DescribeDatabase(ctx context.Context, option client.DescribeDatabaseOption, callOptions ...grpc.CallOption) (*entity.Database, error) {
database, err := mc.mClient.DescribeDatabase(ctx, option, callOptions...)
return database, err
}
// AlterDatabaseProperties alter database properties
func (mc *MilvusClient) AlterDatabaseProperties(ctx context.Context, option client.AlterDatabasePropertiesOption, callOptions ...grpc.CallOption) error {
err := mc.mClient.AlterDatabaseProperties(ctx, option, callOptions...)
return err
}
// DropDatabaseProperties drop database properties
func (mc *MilvusClient) DropDatabaseProperties(ctx context.Context, option client.DropDatabasePropertiesOption, callOptions ...grpc.CallOption) error {
err := mc.mClient.AlterDatabaseProperties(ctx, option, callOptions...)
return err
}
// -- collection --
// CreateCollection Create Collection
@ -144,6 +162,36 @@ func (mc *MilvusClient) DropCollection(ctx context.Context, option client.DropCo
return err
}
// RenameCollection Rename Collection
func (mc *MilvusClient) RenameCollection(ctx context.Context, option client.RenameCollectionOption, callOptions ...grpc.CallOption) error {
err := mc.mClient.RenameCollection(ctx, option, callOptions...)
return err
}
// AlterCollectionProperties Alter collection properties
func (mc *MilvusClient) AlterCollectionProperties(ctx context.Context, option client.AlterCollectionPropertiesOption, callOptions ...grpc.CallOption) error {
err := mc.mClient.AlterCollectionProperties(ctx, option, callOptions...)
return err
}
// DropCollectionProperties Drop collection properties
func (mc *MilvusClient) DropCollectionProperties(ctx context.Context, option client.DropCollectionPropertiesOption, callOptions ...grpc.CallOption) error {
err := mc.mClient.DropCollectionProperties(ctx, option, callOptions...)
return err
}
// AlterCollectionField Alter collection field
func (mc *MilvusClient) AlterCollectionField(ctx context.Context, option client.AlterCollectionFieldPropertiesOption, callOptions ...grpc.CallOption) error {
err := mc.mClient.AlterCollectionFieldProperty(ctx, option, callOptions...)
return err
}
// GetCollectionStats Get collection stats
func (mc *MilvusClient) GetCollectionStats(ctx context.Context, option client.GetCollectionOption) (map[string]string, error) {
stats, err := mc.mClient.GetCollectionStats(ctx, option)
return stats, err
}
// -- partition --
// CreatePartition Create Partition
@ -268,3 +316,9 @@ func (mc *MilvusClient) Get(ctx context.Context, option client.QueryOption, call
resultSet, err := mc.mClient.Get(ctx, option, callOptions...)
return resultSet, err
}
// HybridSearch hybrid search from collection
func (mc *MilvusClient) HybridSearch(ctx context.Context, option client.HybridSearchOption, callOptions ...grpc.CallOption) ([]client.ResultSet, error) {
resultSets, err := mc.mClient.HybridSearch(ctx, option, callOptions...)
return resultSets, err
}

View File

@ -4,33 +4,38 @@ import "github.com/milvus-io/milvus/client/v2/index"
// cost default field name
const (
DefaultInt8FieldName = "int8"
DefaultInt16FieldName = "int16"
DefaultInt32FieldName = "int32"
DefaultInt64FieldName = "int64"
DefaultBoolFieldName = "bool"
DefaultFloatFieldName = "float"
DefaultDoubleFieldName = "double"
DefaultVarcharFieldName = "varchar"
DefaultJSONFieldName = "json"
DefaultArrayFieldName = "array"
DefaultFloatVecFieldName = "floatVec"
DefaultBinaryVecFieldName = "binaryVec"
DefaultFloat16VecFieldName = "fp16Vec"
DefaultBFloat16VecFieldName = "bf16Vec"
DefaultSparseVecFieldName = "sparseVec"
DefaultDynamicNumberField = "dynamicNumber"
DefaultDynamicStringField = "dynamicString"
DefaultDynamicBoolField = "dynamicBool"
DefaultDynamicListField = "dynamicList"
DefaultBoolArrayField = "boolArray"
DefaultInt8ArrayField = "int8Array"
DefaultInt16ArrayField = "int16Array"
DefaultInt32ArrayField = "int32Array"
DefaultInt64ArrayField = "int64Array"
DefaultFloatArrayField = "floatArray"
DefaultDoubleArrayField = "doubleArray"
DefaultVarcharArrayField = "varcharArray"
DefaultInt8FieldName = "int8"
DefaultInt16FieldName = "int16"
DefaultInt32FieldName = "int32"
DefaultInt64FieldName = "int64"
DefaultBoolFieldName = "bool"
DefaultFloatFieldName = "float"
DefaultDoubleFieldName = "double"
DefaultTextFieldName = "text"
DefaultVarcharFieldName = "varchar"
DefaultJSONFieldName = "json"
DefaultArrayFieldName = "array"
DefaultFloatVecFieldName = "floatVec"
DefaultBinaryVecFieldName = "binaryVec"
DefaultFloat16VecFieldName = "fp16Vec"
DefaultBFloat16VecFieldName = "bf16Vec"
DefaultTextSparseVecFieldName = "textSparseVec"
DefaultSparseVecFieldName = "sparseVec"
DefaultDynamicNumberField = "dynamicNumber"
DefaultDynamicStringField = "dynamicString"
DefaultDynamicBoolField = "dynamicBool"
DefaultDynamicListField = "dynamicList"
DefaultBoolArrayField = "boolArray"
DefaultInt8ArrayField = "int8Array"
DefaultInt16ArrayField = "int16Array"
DefaultInt32ArrayField = "int32Array"
DefaultInt64ArrayField = "int64Array"
DefaultFloatArrayField = "floatArray"
DefaultDoubleArrayField = "doubleArray"
DefaultVarcharArrayField = "varcharArray"
DefaultFastPk = "id"
DefaultFastVector = "vector"
)
// cost for test cases
@ -76,3 +81,20 @@ const (
IndexStateFailed index.IndexState = 4
IndexStateRetry index.IndexState = 5
)
// properties
const (
CollectionTTLSeconds = "collection.ttl.seconds"
MmapEnabled = "mmap.enabled"
DatabaseMaxCollections = "database.max.collections"
DatabaseResourceGroups = "database.resource_groups"
DatabaseReplicaNumber = "database.replica.number"
DatabaseForceDenyWriting = "database.force.deny.writing"
DatabaseForceDenyReading = "database.force.deny.reading"
DatabaseDiskQuotaMb = "database.diskQuota.mb"
)
// const for full text search
const (
DefaultTextLang = "en"
)

View File

@ -5,6 +5,7 @@ import (
"fmt"
"math"
"math/rand"
"reflect"
"strings"
"time"
@ -153,3 +154,63 @@ var InvalidExpressions = []InvalidExprStruct{
{Expr: fmt.Sprintf("%s[-1] > %d", DefaultInt8ArrayField, TestCapacity), ErrNil: false, ErrMsg: "cannot parse expression"}, // array[-1] >
{Expr: fmt.Sprintf("%s[-1] > 1", DefaultJSONFieldName), ErrNil: false, ErrMsg: "invalid expression"}, // json[-1] >
}
// Language constants for text generation
const (
English = "en"
Chinese = "zh"
)
func GenText(lang string) string {
englishTopics := []string{
"information retrieval", "data mining", "machine learning",
"natural language processing", "text analysis", "search engines",
"document indexing", "query processing", "relevance ranking",
"semantic search",
}
englishVerbs := []string{
"is", "focuses on", "deals with", "involves", "combines",
"utilizes", "improves", "enables", "enhances", "supports",
}
englishObjects := []string{
"large datasets", "text documents", "user queries", "search results",
"information needs", "relevance scores", "ranking algorithms",
"index structures", "query expansion", "document collections",
}
chineseTopics := []string{
"信息检索", "数据挖掘", "机器学习",
"自然语言处理", "文本分析", "搜索引擎",
"文档索引", "查询处理", "相关性排序",
"语义搜索",
}
chineseVerbs := []string{
"是", "专注于", "处理", "涉及", "结合",
"利用", "改进", "实现", "提升", "支持",
}
chineseObjects := []string{
"大规模数据集", "文本文档", "用户查询", "搜索结果",
"信息需求", "相关性分数", "排序算法",
"索引结构", "查询扩展", "文档集合",
}
var topic, verb, object string
switch lang {
case English:
topic = englishTopics[rand.Intn(len(englishTopics))]
verb = englishVerbs[rand.Intn(len(englishVerbs))]
object = englishObjects[rand.Intn(len(englishObjects))]
return fmt.Sprintf("%s %s %s", topic, verb, object)
case Chinese:
topic = chineseTopics[rand.Intn(len(chineseTopics))]
verb = chineseVerbs[rand.Intn(len(chineseVerbs))]
object = chineseObjects[rand.Intn(len(chineseObjects))]
return fmt.Sprintf("%s%s%s", topic, verb, object)
default:
return "Unsupported language"
}
}
func IsZeroValue(value interface{}) bool {
return reflect.DeepEqual(value, reflect.Zero(reflect.TypeOf(value)).Interface())
}

View File

@ -1,5 +1,3 @@
///go:build L0
package testcases
import (

View File

@ -2,6 +2,7 @@ package testcases
import (
"fmt"
"strconv"
"testing"
"time"
@ -9,6 +10,7 @@ import (
"go.uber.org/zap"
"github.com/milvus-io/milvus/client/v2/entity"
"github.com/milvus-io/milvus/client/v2/index"
client "github.com/milvus-io/milvus/client/v2/milvusclient"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/tests/go_client/common"
@ -40,6 +42,74 @@ func TestCreateCollection(t *testing.T) {
}
}
// fast: create -> index -> load
func TestCreateCollectionFast(t *testing.T) {
// test collection property mmap
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create collection option: WithConsistencyLevel Strong,
collName := common.GenRandomString("alter", 6)
err := mc.CreateCollection(ctx, client.SimpleCreateCollectionOptions(collName, common.DefaultDim))
common.CheckErr(t, err, true)
// verify collection option
coll, _ := mc.DescribeCollection(ctx, client.NewDescribeCollectionOption(collName))
require.Equal(t, entity.FieldTypeInt64, coll.Schema.PKField().DataType)
// load -> insert
prepare, _ := hp.CollPrepare.InsertData(ctx, t, mc, hp.NewInsertParams(coll.Schema), hp.TNewDataOption())
prepare.FlushData(ctx, t, mc, collName)
countRes, err := mc.Query(ctx, client.NewQueryOption(collName).WithFilter("").WithOutputFields(common.QueryCountFieldName))
common.CheckErr(t, err, true)
count, _ := countRes.Fields[0].GetAsInt64(0)
require.EqualValues(t, common.DefaultNb, count)
vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector)
resSearch, err := mc.Search(ctx, client.NewSearchOption(collName, common.DefaultLimit, vectors))
common.CheckErr(t, err, true)
common.CheckSearchResult(t, resSearch, common.DefaultNq, common.DefaultLimit)
}
func TestCreateCollectionFastOption(t *testing.T) {
// test create collection fast with option: ConsistencyLevel, varcharPk, indexOption
// Collection AutoID not works !!!, please set it on the field side~
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create collection option: WithConsistencyLevel Strong,
collName := common.GenRandomString("alter", 6)
index := index.NewHNSWIndex(entity.COSINE, 8, 96)
indexOption := client.NewCreateIndexOption(collName, common.DefaultFastVector, index)
err := mc.CreateCollection(ctx, client.SimpleCreateCollectionOptions(collName, common.DefaultDim).WithDynamicSchema(true).
WithConsistencyLevel(entity.ClStrong).WithIndexOptions(indexOption).WithVarcharPK(true, 10))
common.CheckErr(t, err, true)
// verify collection option
coll, _ := mc.DescribeCollection(ctx, client.NewDescribeCollectionOption(collName))
require.Equal(t, entity.ClStrong, coll.ConsistencyLevel)
require.Equal(t, entity.FieldTypeVarChar, coll.Schema.PKField().DataType)
t.Log("https://github.com/milvus-io/milvus/issues/39524")
// descIdx, _ := mc.DescribeIndex(ctx, client.NewDescribeIndexOption(collName, common.DefaultFastVector))
// common.CheckIndex(t, descIdx, index, common.TNewCheckIndexOpt(common.DefaultNb))
// insert
hp.CollPrepare.InsertData(ctx, t, mc, hp.NewInsertParams(coll.Schema), hp.TNewDataOption())
countRes, err := mc.Query(ctx, client.NewQueryOption(collName).
WithFilter("").WithOutputFields(common.QueryCountFieldName))
common.CheckErr(t, err, true)
count, _ := countRes.Fields[0].GetAsInt64(0)
require.EqualValues(t, common.DefaultNb, count)
vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector)
resSearch, err := mc.Search(ctx, client.NewSearchOption(collName, common.DefaultLimit, vectors).WithOutputFields(common.DefaultDynamicNumberField))
common.CheckErr(t, err, true)
common.CheckSearchResult(t, resSearch, common.DefaultNq, common.DefaultLimit)
common.CheckOutputFields(t, []string{common.DefaultDynamicNumberField}, resSearch[0].Fields)
}
func TestCreateAutoIdCollectionField(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
@ -122,7 +192,7 @@ func TestCreateAutoIdCollectionSchema(t *testing.T) {
// test create auto collection with collection option
func TestCreateAutoIdCollection(t *testing.T) {
t.Skip("waiting for valid AutoId from collection option")
t.Skip("https://github.com/milvus-io/milvus/issues/39523")
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
@ -132,7 +202,7 @@ func TestCreateAutoIdCollection(t *testing.T) {
pkField := entity.NewField().WithName("pk").WithDataType(pkFieldType).WithIsPrimaryKey(true).WithMaxLength(common.MaxLength)
// pk field with name
schema := entity.NewSchema().WithName(collName).WithField(pkField).WithField(vecField)
schema := entity.NewSchema().WithName(collName).WithField(pkField).WithField(vecField).WithAutoID(true)
err := mc.CreateCollection(ctx, client.NewCreateCollectionOption(collName, schema).WithAutoID(true))
common.CheckErr(t, err, true)
@ -777,7 +847,7 @@ func TestCreateVectorWithoutDim(t *testing.T) {
entity.NewField().WithName(vecFieldName).WithDataType(entity.FieldTypeFloatVector),
).WithName(collName)
err := mc.CreateCollection(ctx, client.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, false, fmt.Sprintf("dimension is not defined in field type params of field %s, check type param `dim` for vector field", vecFieldName))
common.CheckErr(t, err, false, "dimension is not defined in field type params")
}
// specify dim for sparse vector -> error
@ -838,7 +908,7 @@ func TestCreateVarcharArrayInvalidLength(t *testing.T) {
for _, invalidLength := range []int64{-1, 0, common.MaxLength + 1} {
arrayVarcharField.WithMaxLength(invalidLength)
err := mc.CreateCollection(ctx, client.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, false, fmt.Sprintf("the maximum length specified for a VarChar field(%s) should be in (0, 65535], but got %d instead: invalid parameter", arrayVarcharField.Name, invalidLength))
common.CheckErr(t, err, false, "the maximum length specified for a VarChar field(array) should be in (0, 65535]")
}
}
@ -860,7 +930,7 @@ func TestCreateVarcharInvalidLength(t *testing.T) {
for _, invalidLength := range []int64{-1, 0, common.MaxLength + 1} {
varcharField.WithMaxLength(invalidLength)
err := mc.CreateCollection(ctx, client.NewCreateCollectionOption(collName, schema))
common.CheckErr(t, err, false, fmt.Sprintf("the maximum length specified for a VarChar field(%s) should be in (0, 65535], but got %d instead", varcharField.Name, invalidLength))
common.CheckErr(t, err, false, "the maximum length specified for a VarChar field(varchar) should be in (0, 65535]")
}
}
@ -934,3 +1004,237 @@ func TestCreateCollectionInvalid(t *testing.T) {
common.CheckErr(t, err, false, mSchema.errMsg)
}
}
// test rename collection
func TestRenameCollection(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create collection
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
// rename collection and verify
newName := common.GenRandomString("new", 6)
err := mc.RenameCollection(ctx, client.NewRenameCollectionOption(schema.CollectionName, newName))
common.CheckErr(t, err, true)
collections, _ := mc.ListCollections(ctx, client.NewListCollectionOption())
require.Contains(t, collections, newName)
require.NotContains(t, collections, schema.CollectionName)
_, err = mc.ListIndexes(ctx, client.NewListIndexOption(schema.CollectionName))
common.CheckErr(t, err, false, "collection not found")
schema.CollectionName = newName
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption())
prepare.FlushData(ctx, t, mc, schema.CollectionName)
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector)
resSearch, errSearch := mc.Search(ctx, client.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors))
common.CheckErr(t, errSearch, true)
common.CheckSearchResult(t, resSearch, common.DefaultNq, common.DefaultLimit)
stats, err := mc.GetCollectionStats(ctx, client.NewGetCollectionStatsOption(newName))
common.CheckErr(t, err, true)
require.Equal(t, map[string]string{common.RowCount: strconv.Itoa(common.DefaultNb)}, stats)
}
// There are collections with the same name in different db. Rename one of them.
func TestRenameCollectionDb(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create collection
collectionName := common.GenRandomString("re", 6)
mc.CreateCollection(ctx, client.SimpleCreateCollectionOptions(collectionName, common.DefaultDim))
// create a database and use database
dbName := common.GenRandomString("db", 4)
mc.CreateDatabase(ctx, client.NewCreateDatabaseOption(dbName))
mc.UseDatabase(ctx, client.NewUseDatabaseOption(dbName))
mc.CreateCollection(ctx, client.SimpleCreateCollectionOptions(collectionName, common.DefaultDim))
// rename db collection rather than default db collection
newName := common.GenRandomString("new", 6)
err := mc.RenameCollection(ctx, client.NewRenameCollectionOption(collectionName, newName))
common.CheckErr(t, err, true)
collections, _ := mc.ListCollections(ctx, client.NewListCollectionOption())
require.Contains(t, collections, newName)
require.NotContains(t, collections, collectionName)
// verify default db collection
mc.UseDatabase(ctx, client.NewUseDatabaseOption(common.DefaultDb))
collectionsDefault, _ := mc.ListCollections(ctx, client.NewListCollectionOption())
require.Contains(t, collectionsDefault, collectionName)
require.NotContains(t, collectionsDefault, newName)
}
func TestRenameCollectionInvalidName(t *testing.T) {
// connect
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2)
mc := createDefaultMilvusClient(ctx, t)
// create collection
collectionName := common.GenRandomString("re", 6)
mc.CreateCollection(ctx, client.SimpleCreateCollectionOptions(collectionName, common.DefaultDim))
// rename collection with invalid name
for _, invalidName := range common.GenInvalidNames() {
log.Debug("TestCreateCollectionWithInvalidFieldName", zap.String("fieldName", invalidName))
err := mc.RenameCollection(ctx, client.NewRenameCollectionOption(collectionName, invalidName))
common.CheckErr(t, err, false, "collection name should not be empty",
"the first character of a collection name must be an underscore or letter",
"collection name can only contain numbers, letters and underscores",
"the length of a collection name must be less than 255 characters",
"collection name can only contain numbers, letters, and underscores")
}
}
func TestRenameCollectionAdvanced(t *testing.T) {
// connect
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2)
mc := createDefaultMilvusClient(ctx, t)
// create 2 collections
name1 := common.GenRandomString("name1", 6)
name2 := common.GenRandomString("name2", 6)
mc.CreateCollection(ctx, client.SimpleCreateCollectionOptions(name1, common.DefaultDim))
mc.CreateCollection(ctx, client.SimpleCreateCollectionOptions(name2, common.DefaultDim))
// rename: old name same with new name
err := mc.RenameCollection(ctx, client.NewRenameCollectionOption(name1, name1))
common.CheckErr(t, err, false, "duplicated new collection name")
// rename to a existed name
err = mc.RenameCollection(ctx, client.NewRenameCollectionOption(name1, name2))
common.CheckErr(t, err, false, "duplicated new collection name")
// rename a not existed collection
err = mc.RenameCollection(ctx, client.NewRenameCollectionOption(common.GenRandomString("a", 2), common.GenRandomString("b", 2)))
common.CheckErr(t, err, false, "collection not found")
}
// alter collection ttl property
func TestCollectionPropertyTtl(t *testing.T) {
// test collection property ttl
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create collection
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption())
prepare.FlushData(ctx, t, mc, schema.CollectionName)
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
res, _ := mc.Query(ctx, client.NewQueryOption(schema.CollectionName).WithFilter("").WithOutputFields(common.QueryCountFieldName))
countBefore, _ := res.GetColumn(common.QueryCountFieldName).GetAsInt64(0)
require.EqualValues(t, common.DefaultNb, countBefore)
err := mc.AlterCollectionProperties(ctx, client.NewAlterCollectionPropertiesOption(schema.CollectionName).WithProperty(common.CollectionTTLSeconds, 2))
common.CheckErr(t, err, true)
coll, _ := mc.DescribeCollection(ctx, client.NewDescribeCollectionOption(schema.CollectionName))
require.Equal(t, map[string]string{common.CollectionTTLSeconds: "2"}, coll.Properties)
time.Sleep(5 * time.Second)
res, _ = mc.Query(ctx, client.NewQueryOption(schema.CollectionName).WithFilter("").WithOutputFields(common.QueryCountFieldName))
countAfter, _ := res.GetColumn(common.QueryCountFieldName).GetAsInt64(0)
require.Contains(t, []int64{0, int64(common.DefaultNb)}, countAfter)
err = mc.DropCollectionProperties(ctx, client.NewDropCollectionPropertiesOption(schema.CollectionName, common.CollectionTTLSeconds))
common.CheckErr(t, err, true)
coll, _ = mc.DescribeCollection(ctx, client.NewDescribeCollectionOption(schema.CollectionName))
require.Equal(t, map[string]string{}, coll.Properties)
}
// create collection with property -> alter property -> writing and reading
func TestCollectionWithPropertyAlterMmap(t *testing.T) {
// test collection property mmap
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create collection
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec),
hp.TNewFieldsOption(), hp.TNewSchemaOption(), hp.TWithProperties(map[string]any{common.MmapEnabled: false}))
coll, _ := mc.DescribeCollection(ctx, client.NewDescribeCollectionOption(schema.CollectionName))
require.Equal(t, map[string]string{common.MmapEnabled: "false"}, coll.Properties)
log.Info("TestCollectionPropertyMmap.DescribeCollection", zap.Any("properties", coll.Properties))
// alter properties
err := mc.AlterCollectionProperties(ctx, client.NewAlterCollectionPropertiesOption(schema.CollectionName).WithProperty(common.MmapEnabled, true))
common.CheckErr(t, err, true)
coll, _ = mc.DescribeCollection(ctx, client.NewDescribeCollectionOption(schema.CollectionName))
require.Equal(t, map[string]string{common.MmapEnabled: "true"}, coll.Properties)
// writing and reading
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption())
prepare.FlushData(ctx, t, mc, schema.CollectionName)
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
res, _ := mc.Query(ctx, client.NewQueryOption(schema.CollectionName).WithFilter("").WithOutputFields(common.QueryCountFieldName))
countBefore, _ := res.GetColumn(common.QueryCountFieldName).GetAsInt64(0)
require.EqualValues(t, common.DefaultNb, countBefore)
}
func TestCollectionPropertyMmap(t *testing.T) {
// test collection property mmap
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create collection
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec),
hp.TNewFieldsOption(), hp.TNewSchemaOption())
// alter properties
err := mc.AlterCollectionProperties(ctx, client.NewAlterCollectionPropertiesOption(schema.CollectionName).WithProperty(common.MmapEnabled, true))
common.CheckErr(t, err, true)
coll, _ := mc.DescribeCollection(ctx, client.NewDescribeCollectionOption(schema.CollectionName))
require.Equal(t, map[string]string{common.MmapEnabled: "true"}, coll.Properties)
// writing and reading
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption())
prepare.FlushData(ctx, t, mc, schema.CollectionName)
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
res, _ := mc.Query(ctx, client.NewQueryOption(schema.CollectionName).WithFilter("").WithOutputFields(common.QueryCountFieldName))
countBefore, _ := res.GetColumn(common.QueryCountFieldName).GetAsInt64(0)
require.EqualValues(t, common.DefaultNb, countBefore)
err = mc.DropCollectionProperties(ctx, client.NewDropCollectionPropertiesOption(schema.CollectionName, common.MmapEnabled))
common.CheckErr(t, err, false, "can not delete mmap properties if collection loaded")
// release collection and drop property
mc.ReleaseCollection(ctx, client.NewReleaseCollectionOption(schema.CollectionName))
err = mc.DropCollectionProperties(ctx, client.NewDropCollectionPropertiesOption(schema.CollectionName, common.MmapEnabled))
common.CheckErr(t, err, true)
coll, _ = mc.DescribeCollection(ctx, client.NewDescribeCollectionOption(schema.CollectionName))
require.Equal(t, map[string]string{}, coll.Properties)
}
func TestCollectionFakeProperties(t *testing.T) {
// test collection property mmap
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create collection with fake property
collName := common.GenRandomString("alter", 6)
err := mc.CreateCollection(ctx, client.SimpleCreateCollectionOptions(collName, common.DefaultDim).WithProperty("1", "bbb"))
common.CheckErr(t, err, true)
coll, _ := mc.DescribeCollection(ctx, client.NewDescribeCollectionOption(collName))
require.Equal(t, map[string]string{"1": "bbb"}, coll.Properties)
// alter collection with fake property
err = mc.AlterCollectionProperties(ctx, client.NewAlterCollectionPropertiesOption(collName).WithProperty("2", 1))
common.CheckErr(t, err, true)
coll, _ = mc.DescribeCollection(ctx, client.NewDescribeCollectionOption(collName))
require.Equal(t, map[string]string{"1": "bbb", "2": "1"}, coll.Properties)
err = mc.DropCollectionProperties(ctx, client.NewDropCollectionPropertiesOption(collName, "ccc"))
common.CheckErr(t, err, true)
coll, _ = mc.DescribeCollection(ctx, client.NewDescribeCollectionOption(collName))
require.Equal(t, map[string]string{"1": "bbb", "2": "1"}, coll.Properties)
}

View File

@ -2,6 +2,7 @@ package testcases
import (
"fmt"
"strconv"
"testing"
"time"
@ -23,10 +24,10 @@ func teardownTest(t *testing.T) func(t *testing.T) {
// drop all db
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
dbs, _ := mc.ListDatabases(ctx, client.NewListDatabaseOption())
dbs, _ := mc.ListDatabase(ctx, client.NewListDatabaseOption())
for _, db := range dbs {
if db != common.DefaultDb {
_ = mc.UsingDatabase(ctx, client.NewUseDatabaseOption(db))
_ = mc.UseDatabase(ctx, client.NewUseDatabaseOption(db))
collections, _ := mc.ListCollections(ctx, client.NewListCollectionOption())
for _, coll := range collections {
_ = mc.DropCollection(ctx, client.NewDropCollectionOption(coll))
@ -50,15 +51,12 @@ func TestDatabase(t *testing.T) {
common.CheckErr(t, err, true)
// list db and verify db1 in dbs
dbs, errList := clientDefault.ListDatabases(ctx, client.NewListDatabaseOption())
dbs, errList := clientDefault.ListDatabase(ctx, client.NewListDatabaseOption())
common.CheckErr(t, errList, true)
require.Containsf(t, dbs, dbName1, fmt.Sprintf("%s db not in dbs: %v", dbName1, dbs))
// new client with db1 -> using db
// new client with db1
clientDB1 := createMilvusClient(ctx, t, &client.ClientConfig{Address: *addr, DBName: dbName1})
t.Log("https://github.com/milvus-io/milvus/issues/34137")
err = clientDB1.UsingDatabase(ctx, client.NewUseDatabaseOption(dbName1))
common.CheckErr(t, err, true)
// create collections -> verify collections contains
_, db1Col1 := hp.CollPrepare.CreateCollection(ctx, t, clientDB1, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
@ -72,24 +70,24 @@ func TestDatabase(t *testing.T) {
dbName2 := common.GenRandomString("db2", 4)
err = clientDefault.CreateDatabase(ctx, client.NewCreateDatabaseOption(dbName2))
common.CheckErr(t, err, true)
dbs, err = clientDefault.ListDatabases(ctx, client.NewListDatabaseOption())
dbs, err = clientDefault.ListDatabase(ctx, client.NewListDatabaseOption())
common.CheckErr(t, err, true)
require.Containsf(t, dbs, dbName2, fmt.Sprintf("%s db not in dbs: %v", dbName2, dbs))
// using db2 -> create collection -> drop collection
err = clientDefault.UsingDatabase(ctx, client.NewUseDatabaseOption(dbName2))
err = clientDefault.UseDatabase(ctx, client.NewUseDatabaseOption(dbName2))
common.CheckErr(t, err, true)
_, db2Col1 := hp.CollPrepare.CreateCollection(ctx, t, clientDefault, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
err = clientDefault.DropCollection(ctx, client.NewDropCollectionOption(db2Col1.CollectionName))
common.CheckErr(t, err, true)
// using empty db -> drop db2
clientDefault.UsingDatabase(ctx, client.NewUseDatabaseOption(""))
clientDefault.UseDatabase(ctx, client.NewUseDatabaseOption(""))
err = clientDefault.DropDatabase(ctx, client.NewDropDatabaseOption(dbName2))
common.CheckErr(t, err, true)
// list db and verify db drop success
dbs, err = clientDefault.ListDatabases(ctx, client.NewListDatabaseOption())
dbs, err = clientDefault.ListDatabase(ctx, client.NewListDatabaseOption())
common.CheckErr(t, err, true)
require.NotContains(t, dbs, dbName2)
@ -98,7 +96,7 @@ func TestDatabase(t *testing.T) {
common.CheckErr(t, err, false, "must drop all collections before drop database")
// drop all db1's collections -> drop db1
clientDB1.UsingDatabase(ctx, client.NewUseDatabaseOption(dbName1))
clientDB1.UseDatabase(ctx, client.NewUseDatabaseOption(dbName1))
err = clientDB1.DropCollection(ctx, client.NewDropCollectionOption(db1Col1.CollectionName))
common.CheckErr(t, err, true)
@ -112,7 +110,7 @@ func TestDatabase(t *testing.T) {
err = clientDefault.DropDatabase(ctx, client.NewDropDatabaseOption(common.DefaultDb))
common.CheckErr(t, err, false, "can not drop default database")
dbs, err = clientDefault.ListDatabases(ctx, client.NewListDatabaseOption())
dbs, err = clientDefault.ListDatabase(ctx, client.NewListDatabaseOption())
common.CheckErr(t, err, true)
require.Containsf(t, dbs, common.DefaultDb, fmt.Sprintf("The db %s not in: %v", common.DefaultDb, dbs))
}
@ -160,7 +158,7 @@ func TestDropDb(t *testing.T) {
common.CheckErr(t, err, true)
// using db and drop the db
err = mc.UsingDatabase(ctx, client.NewUseDatabaseOption(dbName))
err = mc.UseDatabase(ctx, client.NewUseDatabaseOption(dbName))
common.CheckErr(t, err, true)
err = mc.DropDatabase(ctx, client.NewDropDatabaseOption(dbName))
common.CheckErr(t, err, true)
@ -170,7 +168,7 @@ func TestDropDb(t *testing.T) {
common.CheckErr(t, err, false, fmt.Sprintf("database not found[database=%s]", dbName))
// using default db and verify collections
err = mc.UsingDatabase(ctx, client.NewUseDatabaseOption(common.DefaultDb))
err = mc.UseDatabase(ctx, client.NewUseDatabaseOption(common.DefaultDb))
common.CheckErr(t, err, true)
collections, _ = mc.ListCollections(ctx, listCollOpt)
require.Contains(t, collections, defCol.CollectionName)
@ -205,24 +203,23 @@ func TestUsingDb(t *testing.T) {
// using not existed db
dbName := common.GenRandomString("db", 4)
err := mc.UsingDatabase(ctx, client.NewUseDatabaseOption(dbName))
err := mc.UseDatabase(ctx, client.NewUseDatabaseOption(dbName))
common.CheckErr(t, err, false, fmt.Sprintf("database not found[database=%s]", dbName))
// using empty db
err = mc.UsingDatabase(ctx, client.NewUseDatabaseOption(""))
err = mc.UseDatabase(ctx, client.NewUseDatabaseOption(""))
common.CheckErr(t, err, true)
collections, _ = mc.ListCollections(ctx, listCollOpt)
require.Contains(t, collections, col.CollectionName)
// using current db
err = mc.UsingDatabase(ctx, client.NewUseDatabaseOption(common.DefaultDb))
err = mc.UseDatabase(ctx, client.NewUseDatabaseOption(common.DefaultDb))
common.CheckErr(t, err, true)
collections, _ = mc.ListCollections(ctx, listCollOpt)
require.Contains(t, collections, col.CollectionName)
}
func TestClientWithDb(t *testing.T) {
t.Skip("https://github.com/milvus-io/milvus/issues/34137")
teardownSuite := teardownTest(t)
defer teardownSuite(t)
@ -262,7 +259,7 @@ func TestClientWithDb(t *testing.T) {
require.Containsf(t, dbCollections, dbCol1.CollectionName, fmt.Sprintf("The collection %s not in: %v", dbCol1.CollectionName, dbCollections))
// using default db and collection not in
_ = mcDb.UsingDatabase(ctx, client.NewUseDatabaseOption(common.DefaultDb))
_ = mcDb.UseDatabase(ctx, client.NewUseDatabaseOption(common.DefaultDb))
defCollections, _ = mcDb.ListCollections(ctx, listCollOpt)
require.NotContains(t, defCollections, dbCol1.CollectionName)
@ -276,6 +273,166 @@ func TestClientWithDb(t *testing.T) {
require.Contains(t, defCollections, defCol1.CollectionName)
}
func TestAlterDatabase(t *testing.T) {
t.Skip("waiting for AlterDatabase and DescribeDatabase")
func TestDatabasePropertiesCollectionsNum(t *testing.T) {
// create db with properties
teardownSuite := teardownTest(t)
defer teardownSuite(t)
// create db
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
dbName := common.GenRandomString("db", 4)
err := mc.CreateDatabase(ctx, client.NewCreateDatabaseOption(dbName))
common.CheckErr(t, err, true)
// alter database properties
maxCollections := 2
err = mc.AlterDatabaseProperties(ctx, client.NewAlterDatabasePropertiesOption(dbName).WithProperty(common.DatabaseMaxCollections, maxCollections))
common.CheckErr(t, err, true)
// describe database
db, _ := mc.DescribeDatabase(ctx, client.NewDescribeDatabaseOption(dbName))
require.Equal(t, map[string]string{common.DatabaseMaxCollections: strconv.Itoa(maxCollections)}, db.Properties)
require.Equal(t, dbName, db.Name)
// verify properties works
mc.UseDatabase(ctx, client.NewUseDatabaseOption(dbName))
var collections []string
for i := 0; i < maxCollections; i++ {
_, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
collections = append(collections, schema.CollectionName)
}
fields := hp.FieldsFact.GenFieldsForCollection(hp.Int64Vec, hp.TNewFieldsOption())
schema := hp.GenSchema(hp.TNewSchemaOption().TWithFields(fields))
err = mc.CreateCollection(ctx, client.NewCreateCollectionOption(schema.CollectionName, schema))
common.CheckErr(t, err, false, "exceeded the limit number of collections")
// Other db are not restricted by this property
mc.UseDatabase(ctx, client.NewUseDatabaseOption(common.DefaultDb))
for i := 0; i < maxCollections+1; i++ {
hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
}
// drop properties
mc.UseDatabase(ctx, client.NewUseDatabaseOption(dbName))
errDrop := mc.DropDatabaseProperties(ctx, client.NewDropDatabasePropertiesOption(dbName, common.DatabaseMaxCollections))
common.CheckErr(t, errDrop, true)
_, schema1 := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
collections = append(collections, schema1.CollectionName)
// verify collection num
collectionsList, _ := mc.ListCollections(ctx, client.NewListCollectionOption())
require.Subset(t, collectionsList, collections)
require.GreaterOrEqual(t, len(collectionsList), maxCollections)
// describe database after drop properties
db, _ = mc.DescribeDatabase(ctx, client.NewDescribeDatabaseOption(dbName))
require.Equal(t, map[string]string{}, db.Properties)
require.Equal(t, dbName, db.Name)
}
func TestDatabasePropertiesRgReplicas(t *testing.T) {
// create db with properties
teardownSuite := teardownTest(t)
defer teardownSuite(t)
// create db
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
dbName := common.GenRandomString("db", 4)
err := mc.CreateDatabase(ctx, client.NewCreateDatabaseOption(dbName))
common.CheckErr(t, err, true)
// alter database properties
err = mc.AlterDatabaseProperties(ctx, client.NewAlterDatabasePropertiesOption(dbName).
WithProperty(common.DatabaseResourceGroups, "rg1").WithProperty(common.DatabaseReplicaNumber, 2))
common.CheckErr(t, err, true)
// describe database
db, _ := mc.DescribeDatabase(ctx, client.NewDescribeDatabaseOption(dbName))
require.Equal(t, map[string]string{common.DatabaseResourceGroups: "rg1", common.DatabaseReplicaNumber: "2"}, db.Properties)
mc.UseDatabase(ctx, client.NewUseDatabaseOption(dbName))
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true))
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption().TWithNb(1000))
prepare.FlushData(ctx, t, mc, schema.CollectionName)
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
_, err = mc.LoadCollection(ctx, client.NewLoadCollectionOption(schema.CollectionName))
common.CheckErr(t, err, true)
_, err = mc.Query(ctx, client.NewQueryOption(schema.CollectionName).WithLimit(10))
common.CheckErr(t, err, true)
}
func TestDatabasePropertyDeny(t *testing.T) {
t.Skip("https://zilliz.atlassian.net/browse/VDC-7858")
// create db with properties
teardownSuite := teardownTest(t)
defer teardownSuite(t)
// create db and use db
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
dbName := common.GenRandomString("db", 4)
err := mc.CreateDatabase(ctx, client.NewCreateDatabaseOption(dbName))
common.CheckErr(t, err, true)
// alter database properties and check
err = mc.AlterDatabaseProperties(ctx, client.NewAlterDatabasePropertiesOption(dbName).
WithProperty(common.DatabaseForceDenyWriting, true).
WithProperty(common.DatabaseForceDenyReading, true))
common.CheckErr(t, err, true)
db, _ := mc.DescribeDatabase(ctx, client.NewDescribeDatabaseOption(dbName))
require.Equal(t, map[string]string{common.DatabaseForceDenyWriting: "true", common.DatabaseForceDenyReading: "true"}, db.Properties)
err = mc.UseDatabase(ctx, client.NewUseDatabaseOption(dbName))
common.CheckErr(t, err, true)
// prepare collection: create -> index -> load
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true))
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// reading
_, err = mc.Query(ctx, client.NewQueryOption(schema.CollectionName).WithLimit(10))
common.CheckErr(t, err, false, "access has been disabled by the administrator")
// writing
columns, _ := hp.GenColumnsBasedSchema(schema, hp.TNewDataOption().TWithNb(10))
_, err = mc.Insert(ctx, client.NewColumnBasedInsertOption(schema.CollectionName, columns...))
common.CheckErr(t, err, false, "access has been disabled by the administrator")
}
func TestDatabaseFakeProperties(t *testing.T) {
// create db with properties
teardownSuite := teardownTest(t)
defer teardownSuite(t)
// create db
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
dbName := common.GenRandomString("db", 4)
err := mc.CreateDatabase(ctx, client.NewCreateDatabaseOption(dbName))
common.CheckErr(t, err, true)
// alter database with useless properties
properties := map[string]any{
"key_1": 1,
"key2": 1.9,
"key-3": true,
"key.4": "a.b.c",
}
for key, value := range properties {
err = mc.AlterDatabaseProperties(ctx, client.NewAlterDatabasePropertiesOption(dbName).WithProperty(key, value))
common.CheckErr(t, err, true)
}
// describe database
db, _ := mc.DescribeDatabase(ctx, client.NewDescribeDatabaseOption(dbName))
require.EqualValues(t, map[string]string{"key_1": "1", "key2": "1.9", "key-3": "true", "key.4": "a.b.c"}, db.Properties)
// drop database properties
err = mc.DropDatabaseProperties(ctx, client.NewDropDatabasePropertiesOption(dbName, "aaa"))
common.CheckErr(t, err, true)
}

View File

@ -2,6 +2,7 @@ package testcases
import (
"fmt"
"sync"
"testing"
"time"
@ -440,7 +441,6 @@ func TestDeleteComplexExpr(t *testing.T) {
expr string
count int
}
capacity := common.TestCapacity
exprLimits := []exprCount{
{expr: fmt.Sprintf("%s >= 1000 || %s > 2000", common.DefaultInt64FieldName, common.DefaultInt64FieldName), count: 2000},
@ -463,7 +463,58 @@ func TestDeleteComplexExpr(t *testing.T) {
// data type not match and no error
{expr: fmt.Sprintf("%s['number'] == '0' ", common.DefaultJSONFieldName), count: 0},
}
ch := make(chan struct{}, 5)
wg := sync.WaitGroup{}
testFunc := func(exprLimit exprCount) {
defer func() {
wg.Done()
<-ch
}()
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2)
mc := createDefaultMilvusClient(ctx, t)
// create collection and a partition
cp := hp.NewCreateCollectionParams(hp.Int64VecAllScalar)
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, cp, hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true))
// insert [0, 3000) into default
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption().TWithMaxCapacity(common.TestCapacity))
prepare.FlushData(ctx, t, mc, schema.CollectionName)
// index and load
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
log.Debug("TestDeleteComplexExpr", zap.Any("expr", exprLimit.expr))
resDe, err := mc.Delete(ctx, client.NewDeleteOption(schema.CollectionName).WithExpr(exprLimit.expr))
common.CheckErr(t, err, true)
log.Debug("delete count", zap.Bool("equal", int64(exprLimit.count) == resDe.DeleteCount))
// require.Equal(t, int64(exprLimit.count), resDe.DeleteCount)
resQuery, err := mc.Query(ctx, client.NewQueryOption(schema.CollectionName).WithFilter(exprLimit.expr).WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
require.Zero(t, resQuery.ResultCount)
}
for _, exprLimit := range exprLimits {
exprLimit := exprLimit
ch <- struct{}{}
wg.Add(1)
go testFunc(exprLimit)
}
wg.Wait()
}
// test delete json expr
func TestDeleteComplexExprJson(t *testing.T) {
type exprCount struct {
expr string
count int
}
capacity := common.TestCapacity
exprLimits := []exprCount{
// json field
{expr: fmt.Sprintf("%s > 1499.5", common.DefaultJSONFieldName), count: 1500 / 2}, // json >= 1500.0
{expr: fmt.Sprintf("%s like '21%%'", common.DefaultJSONFieldName), count: 100 / 4}, // json like '21%'

View File

@ -0,0 +1,283 @@
package testcases
import (
"testing"
"time"
"github.com/milvus-io/milvus/client/v2/entity"
"github.com/milvus-io/milvus/client/v2/index"
"github.com/milvus-io/milvus/client/v2/milvusclient"
"github.com/milvus-io/milvus/tests/go_client/common"
hp "github.com/milvus-io/milvus/tests/go_client/testcases/helper"
)
func TestFullTextSearchDefault(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create -> insert -> flush -> index -> load
analyzerParams := map[string]any{"tokenizer": "standard"}
fieldsOption := hp.TNewFieldsOption().TWithAnalyzerParams(analyzerParams)
function := hp.TNewBM25Function(common.DefaultTextFieldName, common.DefaultTextSparseVecFieldName)
schemaOption := hp.TNewSchemaOption().TWithFunction(function)
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.FullTextSearch), fieldsOption, schemaOption)
insertOption := hp.TNewDataOption().TWithTextLang(common.DefaultTextLang)
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), insertOption)
prepare.FlushData(ctx, t, mc, schema.CollectionName)
indexparams := hp.TNewIndexParams(schema).TWithFieldIndex(map[string]index.Index{common.DefaultTextSparseVecFieldName: index.NewSparseInvertedIndex(entity.BM25, 0.1)})
prepare.CreateIndex(ctx, t, mc, indexparams)
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// search
queries := hp.GenFullTextQuery(common.DefaultNq, common.DefaultTextLang)
vectors := make([]entity.Vector, 0, len(queries))
for _, query := range queries {
vectors = append(vectors, entity.Text(query))
}
resSearch, err := mc.Search(ctx, milvusclient.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
common.CheckSearchResult(t, resSearch, common.DefaultNq, common.DefaultLimit)
}
// TestSearchFullTextBase tests basic full text search functionality with different languages
func TestSearchFullTextWithDiffLang(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// Test cases for different languages and analyzers
testCases := []struct {
name string
language string
analyzer string
query string
numRows int
topK int
}{
{
name: "English_Standard",
language: "english",
analyzer: "standard",
query: "what is information retrieval and its applications?",
numRows: 3000,
topK: 10,
},
{
name: "Chinese_Jieba",
language: "chinese",
analyzer: "jieba",
query: "信息检索的应用",
numRows: 3000,
topK: 10,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
analyzerParams := map[string]any{"tokenizer": tc.analyzer}
fieldsOption := hp.TNewFieldsOption().TWithAnalyzerParams(analyzerParams)
function := hp.TNewBM25Function(common.DefaultTextFieldName, common.DefaultTextSparseVecFieldName)
schemaOption := hp.TNewSchemaOption().TWithFunction(function)
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.FullTextSearch), fieldsOption, schemaOption)
insertOption := hp.TNewDataOption().TWithTextLang(tc.language).TWithNb(tc.numRows)
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), insertOption)
prepare.FlushData(ctx, t, mc, schema.CollectionName)
indexparams := hp.TNewIndexParams(schema).TWithFieldIndex(map[string]index.Index{common.DefaultTextSparseVecFieldName: index.NewSparseInvertedIndex(entity.BM25, 0.1)})
prepare.CreateIndex(ctx, t, mc, indexparams)
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// search
queries := []string{tc.query}
vectors := make([]entity.Vector, 0, len(queries))
for _, query := range queries {
vectors = append(vectors, entity.Text(query))
}
resSearch, err := mc.Search(ctx, milvusclient.NewSearchOption(schema.CollectionName, tc.topK, vectors).WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
common.CheckSearchResult(t, resSearch, len(queries), tc.topK)
})
}
}
// TestSearchFullTextWithDynamicField tests full text search with dynamic field enabled
func TestSearchFullTextWithDynamicField(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// Test cases for different languages and analyzers
testCases := []struct {
name string
language string
analyzer string
query string
numRows int
topK int
}{
{
name: "English_Standard",
language: "english",
analyzer: "standard",
query: "what is information retrieval and its applications?",
numRows: 1000,
topK: 5,
},
{
name: "Chinese_Jieba",
language: "chinese",
analyzer: "jieba",
query: "信息检索的应用",
numRows: 1000,
topK: 5,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
analyzerParams := map[string]any{"tokenizer": tc.analyzer}
fieldsOption := hp.TNewFieldsOption().TWithAnalyzerParams(analyzerParams)
function := hp.TNewBM25Function(common.DefaultTextFieldName, common.DefaultTextSparseVecFieldName)
schemaOption := hp.TNewSchemaOption().TWithFunction(function).TWithEnableDynamicField(true)
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.FullTextSearch), fieldsOption, schemaOption)
insertOption := hp.TNewDataOption().TWithTextLang(tc.language).TWithNb(tc.numRows)
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), insertOption)
prepare.FlushData(ctx, t, mc, schema.CollectionName)
indexparams := hp.TNewIndexParams(schema).TWithFieldIndex(map[string]index.Index{common.DefaultTextSparseVecFieldName: index.NewSparseInvertedIndex(entity.BM25, 0.1)})
prepare.CreateIndex(ctx, t, mc, indexparams)
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// search
queries := []string{tc.query}
vectors := make([]entity.Vector, 0, len(queries))
for _, query := range queries {
vectors = append(vectors, entity.Text(query))
}
resSearch, err := mc.Search(ctx, milvusclient.NewSearchOption(schema.CollectionName, tc.topK, vectors).WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
common.CheckSearchResult(t, resSearch, len(queries), tc.topK)
})
}
}
// TestSearchFullTextWithPartitionKey tests full text search with partition key
func TestSearchFullTextWithPartitionKey(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// Test cases for different languages and analyzers
testCases := []struct {
name string
language string
analyzer string
query string
numRows int
topK int
}{
{
name: "English_Standard",
language: "english",
analyzer: "standard",
query: "what is information retrieval and its applications?",
numRows: 1000,
topK: 5,
},
{
name: "Chinese_Jieba",
language: "chinese",
analyzer: "jieba",
query: "信息检索的应用",
numRows: 1000,
topK: 5,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
analyzerParams := map[string]any{"tokenizer": tc.analyzer}
fieldsOption := hp.TNewFieldsOption().TWithAnalyzerParams(analyzerParams).TWithIsPartitionKey(true)
function := hp.TNewBM25Function(common.DefaultTextFieldName, common.DefaultTextSparseVecFieldName)
schemaOption := hp.TNewSchemaOption().TWithFunction(function)
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.FullTextSearch), fieldsOption, schemaOption)
insertOption := hp.TNewDataOption().TWithTextLang(tc.language).TWithNb(tc.numRows)
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), insertOption)
prepare.FlushData(ctx, t, mc, schema.CollectionName)
indexparams := hp.TNewIndexParams(schema).TWithFieldIndex(map[string]index.Index{common.DefaultTextSparseVecFieldName: index.NewSparseInvertedIndex(entity.BM25, 0.1)})
prepare.CreateIndex(ctx, t, mc, indexparams)
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// search
queries := []string{tc.query}
vectors := make([]entity.Vector, 0, len(queries))
for _, query := range queries {
vectors = append(vectors, entity.Text(query))
}
resSearch, err := mc.Search(ctx, milvusclient.NewSearchOption(schema.CollectionName, tc.topK, vectors).WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
common.CheckSearchResult(t, resSearch, len(queries), tc.topK)
})
}
}
// TestSearchFullTextWithEmptyData tests full text search with empty data
func TestSearchFullTextWithEmptyData(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// Test cases for different empty percent
testCases := []struct {
name string
language string
analyzer string
query string
numRows int
topK int
emptyPercent int
}{
{
name: "English_Standard",
language: "english",
analyzer: "standard",
query: "what is information retrieval and its applications?",
numRows: 3000,
topK: 5,
emptyPercent: 50,
},
{
name: "Chinese_Jieba",
language: "chinese",
analyzer: "jieba",
query: "信息检索的应用",
numRows: 3000,
topK: 5,
emptyPercent: 80,
},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
analyzerParams := map[string]any{"tokenizer": tc.analyzer}
fieldsOption := hp.TNewFieldsOption().TWithAnalyzerParams(analyzerParams).TWithIsPartitionKey(true)
function := hp.TNewBM25Function(common.DefaultTextFieldName, common.DefaultTextSparseVecFieldName)
schemaOption := hp.TNewSchemaOption().TWithFunction(function)
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.FullTextSearch), fieldsOption, schemaOption)
insertOption := hp.TNewDataOption().TWithTextLang(tc.language).TWithNb(tc.numRows).TWithTextEmptyPercent(tc.emptyPercent)
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), insertOption)
prepare.FlushData(ctx, t, mc, schema.CollectionName)
indexparams := hp.TNewIndexParams(schema).TWithFieldIndex(map[string]index.Index{common.DefaultTextSparseVecFieldName: index.NewSparseInvertedIndex(entity.BM25, 0.1)})
prepare.CreateIndex(ctx, t, mc, indexparams)
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// search
queries := []string{tc.query}
vectors := make([]entity.Vector, 0, len(queries))
for _, query := range queries {
vectors = append(vectors, entity.Text(query))
}
resSearch, err := mc.Search(ctx, milvusclient.NewSearchOption(schema.CollectionName, tc.topK, vectors).WithConsistencyLevel(entity.ClStrong))
common.CheckErr(t, err, true)
common.CheckSearchResult(t, resSearch, len(queries), tc.topK)
})
}
}

View File

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"log"
"sync"
"testing"
"time"
@ -91,7 +92,15 @@ func prepareDataForGroupBySearch(t *testing.T, loopInsert int, insertNi int, idx
// output_fields: pk + groupBy
func TestSearchGroupByFloatDefault(t *testing.T) {
t.Parallel()
for _, idx := range genGroupByVectorIndex(entity.L2) {
concurrency := 10
ch := make(chan struct{}, concurrency)
wg := sync.WaitGroup{}
testFunc := func(idx index.Index) {
defer func() {
wg.Done()
<-ch
}()
// prepare data
mc, ctx, collName := prepareDataForGroupBySearch(t, 100, 200, idx, false)
@ -144,11 +153,25 @@ func TestSearchGroupByFloatDefault(t *testing.T) {
}
}
}
for _, idx := range genGroupByVectorIndex(entity.L2) {
ch <- struct{}{}
wg.Add(1)
go testFunc(idx)
}
wg.Wait()
}
func TestSearchGroupByFloatDefaultCosine(t *testing.T) {
t.Parallel()
for _, idx := range genGroupByVectorIndex(entity.COSINE) {
concurrency := 10
ch := make(chan struct{}, concurrency)
wg := sync.WaitGroup{}
testFunc := func(idx index.Index) {
defer func() {
wg.Done()
<-ch
}()
// prepare data
mc, ctx, collName := prepareDataForGroupBySearch(t, 100, 200, idx, false)
@ -196,6 +219,12 @@ func TestSearchGroupByFloatDefaultCosine(t *testing.T) {
}
}
}
for _, idx := range genGroupByVectorIndex(entity.COSINE) {
ch <- struct{}{}
wg.Add(1)
go testFunc(idx)
}
wg.Wait()
}
// test groupBy search sparse vector
@ -327,7 +356,16 @@ func TestSearchGroupByBinaryGrowing(t *testing.T) {
// groupBy in growing segments, maybe growing index or brute force
func TestSearchGroupByFloatGrowing(t *testing.T) {
t.Parallel()
for _, metricType := range hp.SupportFloatMetricType {
concurrency := 10
ch := make(chan struct{}, concurrency)
wg := sync.WaitGroup{}
testFunc := func(metricType entity.MetricType) {
defer func() {
wg.Done()
<-ch
}()
idxHnsw := index.NewHNSWIndex(metricType, 8, 96)
mc, ctx, collName := prepareDataForGroupBySearch(t, 100, 200, idxHnsw, true)
@ -376,6 +414,13 @@ func TestSearchGroupByFloatGrowing(t *testing.T) {
}
}
}
for _, metricType := range hp.SupportFloatMetricType {
ch <- struct{}{}
wg.Add(1)
go testFunc(metricType)
}
wg.Wait()
}
// groupBy + pagination

View File

@ -1,5 +1,9 @@
package helper
import (
"github.com/milvus-io/milvus/client/v2/entity"
)
type CreateCollectionParams struct {
CollectionFieldsType CollectionFieldsType // collection fields type
}
@ -9,3 +13,25 @@ func NewCreateCollectionParams(collectionFieldsType CollectionFieldsType) *Creat
CollectionFieldsType: collectionFieldsType,
}
}
type CreateCollectionOpt func(opt *createCollectionOpt)
type createCollectionOpt struct {
shardNum int32
enabledDynamicSchema bool
consistencyLevel entity.ConsistencyLevel
properties map[string]any
}
func TWithShardNum(shardNum int32) CreateCollectionOpt {
return func(opt *createCollectionOpt) {
opt.shardNum = shardNum
}
}
func TWithProperties(properties map[string]any) CreateCollectionOpt {
return func(opt *createCollectionOpt) {
opt.properties = properties
}
}

View File

@ -3,7 +3,11 @@ package helper
import (
"bytes"
"encoding/json"
"fmt"
"math/rand"
"slices"
"strconv"
"strings"
"go.uber.org/zap"
@ -38,14 +42,16 @@ func (opt *InsertParams) TWithIsRows(isRows bool) *InsertParams {
// GenColumnDataOption -- create column data --
type GenDataOption struct {
nb int
start int
dim int
maxLen int
sparseMaxLen int
maxCapacity int
elementType entity.FieldType
fieldName string
nb int
start int
dim int
maxLen int
sparseMaxLen int
maxCapacity int
elementType entity.FieldType
fieldName string
textLang string
textEmptyPercent int
}
func (opt *GenDataOption) TWithNb(nb int) *GenDataOption {
@ -88,15 +94,28 @@ func (opt *GenDataOption) TWithElementType(eleType entity.FieldType) *GenDataOpt
return opt
}
func (opt *GenDataOption) TWithTextLang(lang string) *GenDataOption {
opt.textLang = lang
return opt
}
func (opt *GenDataOption) TWithTextEmptyPercent(percent int) *GenDataOption {
opt.textEmptyPercent = percent
return opt
}
func TNewDataOption() *GenDataOption {
return &GenDataOption{
nb: common.DefaultNb,
start: 0,
dim: common.DefaultDim,
maxLen: common.TestMaxLen,
sparseMaxLen: common.TestMaxLen,
maxCapacity: common.TestCapacity,
elementType: entity.FieldTypeNone,
nb: common.DefaultNb,
start: 0,
dim: common.DefaultDim,
maxLen: common.TestMaxLen,
sparseMaxLen: common.TestMaxLen,
maxCapacity: common.TestCapacity,
elementType: entity.FieldTypeNone,
fieldName: "",
textLang: "",
textEmptyPercent: 0,
}
}
@ -197,10 +216,13 @@ func GenArrayColumnData(nb int, eleType entity.FieldType, option GenDataOption)
}
type JSONStruct struct {
Number int32 `json:"number,omitempty" milvus:"name:number"`
String string `json:"string,omitempty" milvus:"name:string"`
Number int32 `json:"number,omitempty" milvus:"name:number"`
String string `json:"string,omitempty" milvus:"name:string"`
Float float32 `json:"float,omitempty" milvus:"name:float"`
*BoolStruct
List []int64 `json:"list,omitempty" milvus:"name:list"`
List []int64 `json:"list,omitempty" milvus:"name:list"`
FloatArray []float64 `json:"floatArray,omitempty" milvus:"name:floatArray"`
StringArray []string `json:"stringArray,omitempty" milvus:"name:stringArray"`
}
// GenDefaultJSONData gen default column with data
@ -216,12 +238,15 @@ func GenDefaultJSONData(nb int, option GenDataOption) [][]byte {
if i < (start+nb)/2 {
if i%2 == 0 {
m = JSONStruct{
String: strconv.Itoa(i),
BoolStruct: _bool,
String: strconv.Itoa(i),
BoolStruct: _bool,
FloatArray: []float64{float64(i), float64(i), float64(i)},
StringArray: []string{fmt.Sprintf("%05d", i)},
}
} else {
m = JSONStruct{
Number: int32(i),
Float: float32(i),
String: strconv.Itoa(i),
BoolStruct: _bool,
List: []int64{int64(i), int64(i + 1)},
@ -249,6 +274,24 @@ func GenDefaultJSONData(nb int, option GenDataOption) [][]byte {
return jsonValues
}
func GenNestedJSON(depth int, value any) map[string]interface{} {
if depth == 1 {
return map[string]interface{}{"value": value}
}
return map[string]interface{}{
fmt.Sprintf("level%d", depth): GenNestedJSON(depth-1, value),
}
}
func GenNestedJSONExprKey(depth int, jsonField string) string {
var pathParts []string
for i := depth; i > 1; i-- {
pathParts = append(pathParts, fmt.Sprintf("level%d", i))
}
pathParts = append(pathParts, "value")
return fmt.Sprintf("%s['%s']", jsonField, strings.Join(pathParts, "']['"))
}
// GenColumnData GenColumnDataOption except dynamic column
func GenColumnData(nb int, fieldType entity.FieldType, option GenDataOption) column.Column {
dim := option.dim
@ -310,8 +353,35 @@ func GenColumnData(nb int, fieldType entity.FieldType, option GenDataOption) col
case entity.FieldTypeVarChar:
varcharValues := make([]string, 0, nb)
for i := start; i < start+nb; i++ {
varcharValues = append(varcharValues, strconv.Itoa(i))
if option.textLang != "" {
// Use language-specific text generation
var lang string
switch option.textLang {
case "en", "english":
lang = "en"
case "zh", "chinese":
lang = "zh"
default:
// Fallback to sequential numbers for unsupported languages
for i := start; i < start+nb; i++ {
varcharValues = append(varcharValues, strconv.Itoa(i))
}
return column.NewColumnVarChar(fieldName, varcharValues)
}
// Generate text data with empty values based on textEmptyPercent
for i := 0; i < nb; i++ {
if rand.Float64()*100 < float64(option.textEmptyPercent) {
varcharValues = append(varcharValues, "")
} else {
varcharValues = append(varcharValues, common.GenText(lang))
}
}
} else {
// Default behavior: sequential numbers
for i := start; i < start+nb; i++ {
varcharValues = append(varcharValues, strconv.Itoa(i))
}
}
return column.NewColumnVarChar(fieldName, varcharValues)
@ -449,6 +519,16 @@ func MergeColumnsToDynamic(nb int, columns []column.Column, columnName string) *
return jsonColumn
}
func GetBm25FunctionsOutputFields(schema *entity.Schema) []string {
var outputFields []string
for _, fn := range schema.Functions {
if fn.Type == entity.FunctionTypeBM25 {
outputFields = append(outputFields, fn.OutputFieldNames...)
}
}
return outputFields
}
func GenColumnsBasedSchema(schema *entity.Schema, option *GenDataOption) ([]column.Column, []column.Column) {
if nil == schema || schema.CollectionName == "" {
log.Fatal("[GenColumnsBasedSchema] Nil Schema is not expected")
@ -463,6 +543,16 @@ func GenColumnsBasedSchema(schema *entity.Schema, option *GenDataOption) ([]colu
if field.AutoID {
continue
}
option.fieldName = field.Name
if option.fieldName == "" {
option.fieldName = field.Name
}
if slices.Contains(GetBm25FunctionsOutputFields(schema), field.Name) {
continue
}
log.Info("GenColumnsBasedSchema", zap.Any("field", field))
// set field name to option
option.TWithFieldName(field.Name)
columns = append(columns, GenColumnData(option.nb, field.DataType, *option))
}
if schema.EnableDynamicField {

View File

@ -107,6 +107,7 @@ const (
Int64MultiVec CollectionFieldsType = 6 // int64 + floatVec + binaryVec + fp16Vec + bf16vec
AllFields CollectionFieldsType = 7 // all fields excepted sparse
Int64VecAllScalar CollectionFieldsType = 8 // int64 + floatVec + all scalar fields
FullTextSearch CollectionFieldsType = 9 // int64 + varchar + sparse vector + analyzer + function
)
type GenFieldsOption struct {
@ -116,6 +117,8 @@ type GenFieldsOption struct {
MaxLength int64 // varchar len or array capacity
MaxCapacity int64
IsPartitionKey bool
EnableAnalyzer bool
AnalyzerParams map[string]any
ElementType entity.FieldType
}
@ -127,6 +130,8 @@ func TNewFieldsOption() *GenFieldsOption {
MaxCapacity: common.TestCapacity,
IsDynamic: false,
IsPartitionKey: false,
EnableAnalyzer: false,
AnalyzerParams: make(map[string]any),
ElementType: entity.FieldTypeNone,
}
}
@ -166,6 +171,16 @@ func (opt *GenFieldsOption) TWithMaxCapacity(maxCapacity int64) *GenFieldsOption
return opt
}
func (opt *GenFieldsOption) TWithEnableAnalyzer(enableAnalyzer bool) *GenFieldsOption {
opt.EnableAnalyzer = enableAnalyzer
return opt
}
func (opt *GenFieldsOption) TWithAnalyzerParams(analyzerParams map[string]any) *GenFieldsOption {
opt.AnalyzerParams = analyzerParams
return opt
}
// factory
type FieldsFactory struct{}
@ -341,6 +356,23 @@ func (cf FieldsInt64VecAllScalar) GenFields(option GenFieldsOption) []*entity.Fi
return fields
}
type FieldsFullTextSearch struct{}
func (cf FieldsFullTextSearch) GenFields(option GenFieldsOption) []*entity.Field {
pkField := entity.NewField().WithName(GetFieldNameByFieldType(entity.FieldTypeInt64)).WithDataType(entity.FieldTypeInt64).WithIsPrimaryKey(true)
textField := entity.NewField().WithName(common.DefaultTextFieldName).WithDataType(entity.FieldTypeVarChar).WithMaxLength(option.MaxLength).WithIsPartitionKey(option.IsPartitionKey).WithEnableAnalyzer(true).WithAnalyzerParams(option.AnalyzerParams)
sparseVecField := entity.NewField().WithName(common.DefaultTextSparseVecFieldName).WithDataType(entity.FieldTypeSparseVector)
if option.AutoID {
pkField.WithIsAutoID(option.AutoID)
}
fields := []*entity.Field{
pkField,
textField,
sparseVecField,
}
return fields
}
func (ff FieldsFactory) GenFieldsForCollection(collectionFieldsType CollectionFieldsType, option *GenFieldsOption) []*entity.Field {
log.Info("GenFieldsForCollection", zap.Any("GenFieldsOption", option))
switch collectionFieldsType {
@ -360,6 +392,8 @@ func (ff FieldsFactory) GenFieldsForCollection(collectionFieldsType CollectionFi
return FieldsAllFields{}.GenFields(*option)
case Int64VecAllScalar:
return FieldsInt64VecAllScalar{}.GenFields(*option)
case FullTextSearch:
return FieldsFullTextSearch{}.GenFields(*option)
default:
return FieldsInt64Vec{}.GenFields(*option)
}

View File

@ -0,0 +1,14 @@
package helper
import (
"github.com/milvus-io/milvus/client/v2/entity"
)
// TNewBM25Function creates a new BM25 function with the given input and output fields
func TNewBM25Function(inputField, outputField string) *entity.Function {
return entity.NewFunction().
WithName(inputField + "_bm25_emb").
WithInputFields(inputField).
WithOutputFields(outputField).
WithType(entity.FunctionTypeBM25)
}

View File

@ -104,7 +104,7 @@ func GetInvalidPartitionKeyFieldType() []entity.FieldType {
return nonPkFieldTypes
}
// ----------------- prepare data --------------------------
// CollectionPrepare ----------------- prepare data --------------------------
type CollectionPrepare struct{}
var (
@ -112,14 +112,44 @@ var (
FieldsFact FieldsFactory
)
func mergeOptions(schema *entity.Schema, opts ...CreateCollectionOpt) clientv2.CreateCollectionOption {
//
collectionOption := clientv2.NewCreateCollectionOption(schema.CollectionName, schema)
tmpOption := &createCollectionOpt{}
for _, o := range opts {
o(tmpOption)
}
if !common.IsZeroValue(tmpOption.shardNum) {
collectionOption.WithShardNum(tmpOption.shardNum)
}
if !common.IsZeroValue(tmpOption.enabledDynamicSchema) {
collectionOption.WithDynamicSchema(tmpOption.enabledDynamicSchema)
}
if !common.IsZeroValue(tmpOption.properties) {
for k, v := range tmpOption.properties {
collectionOption.WithProperty(k, v)
}
}
if !common.IsZeroValue(tmpOption.consistencyLevel) {
collectionOption.WithConsistencyLevel(tmpOption.consistencyLevel)
}
return collectionOption
}
func (chainTask *CollectionPrepare) CreateCollection(ctx context.Context, t *testing.T, mc *base.MilvusClient,
cp *CreateCollectionParams, fieldOpt *GenFieldsOption, schemaOpt *GenSchemaOption,
cp *CreateCollectionParams, fieldOpt *GenFieldsOption, schemaOpt *GenSchemaOption, opts ...CreateCollectionOpt,
) (*CollectionPrepare, *entity.Schema) {
fields := FieldsFact.GenFieldsForCollection(cp.CollectionFieldsType, fieldOpt)
schemaOpt.Fields = fields
schema := GenSchema(schemaOpt)
err := mc.CreateCollection(ctx, clientv2.NewCreateCollectionOption(schema.CollectionName, schema))
createCollectionOption := mergeOptions(schema, opts...)
err := mc.CreateCollection(ctx, createCollectionOption)
common.CheckErr(t, err, true)
t.Cleanup(func() {
@ -141,6 +171,8 @@ func (chainTask *CollectionPrepare) InsertData(ctx context.Context, t *testing.T
if nil == ip.Schema || ip.Schema.CollectionName == "" {
log.Fatal("[InsertData] Nil Schema is not expected")
}
// print option
log.Info("GenDataOption", zap.Any("option", option))
columns, dynamicColumns := GenColumnsBasedSchema(ip.Schema, option)
insertOpt := clientv2.NewColumnBasedInsertOption(ip.Schema.CollectionName).WithColumns(columns...).WithColumns(dynamicColumns...)
if ip.PartitionName != "" {

View File

@ -55,6 +55,10 @@ var SupportBinIvfFlatMetricType = []entity.MetricType{
entity.HAMMING,
}
var SupportFullTextSearchMetricsType = []entity.MetricType{
entity.BM25,
}
var UnsupportedSparseVecMetricsType = []entity.MetricType{
entity.L2,
entity.COSINE,

View File

@ -66,6 +66,14 @@ func GenSearchVectors(nq int, dim int, dataType entity.FieldType) []entity.Vecto
return vectors
}
func GenFullTextQuery(nq int, lang string) []string {
queries := make([]string, 0, nq)
for i := 0; i < nq; i++ {
queries = append(queries, common.GenText(lang))
}
return queries
}
func GenFp16OrBf16VectorsFromFloatVector(nq int, dim int, dataType entity.FieldType) []entity.Vector {
vectors := make([]entity.Vector, 0, nq)
switch dataType {

View File

@ -12,6 +12,7 @@ type GenSchemaOption struct {
AutoID bool
Fields []*entity.Field
EnableDynamicField bool
Function *entity.Function
}
func TNewSchemaOption() *GenSchemaOption {
@ -43,6 +44,11 @@ func (opt *GenSchemaOption) TWithFields(fields []*entity.Field) *GenSchemaOption
return opt
}
func (opt *GenSchemaOption) TWithFunction(function *entity.Function) *GenSchemaOption {
opt.Function = function
return opt
}
func GenSchema(option *GenSchemaOption) *entity.Schema {
if len(option.Fields) == 0 {
log.Fatal("Require at least a primary field and a vector field")
@ -64,5 +70,8 @@ func GenSchema(option *GenSchemaOption) *entity.Schema {
if option.EnableDynamicField {
schema.WithDynamicFieldEnabled(option.EnableDynamicField)
}
if option.Function != nil {
schema.WithFunction(option.Function)
}
return schema
}

View File

@ -0,0 +1,429 @@
package testcases
import (
"fmt"
"sync"
"testing"
"time"
"github.com/stretchr/testify/require"
"github.com/milvus-io/milvus/client/v2/column"
"github.com/milvus-io/milvus/client/v2/entity"
"github.com/milvus-io/milvus/client/v2/index"
client "github.com/milvus-io/milvus/client/v2/milvusclient"
"github.com/milvus-io/milvus/tests/go_client/common"
hp "github.com/milvus-io/milvus/tests/go_client/testcases/helper"
)
func TestHybridSearchDefault(t *testing.T) {
t.Parallel()
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create -> insert [0, 3000) -> flush -> index -> load
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption())
prepare.FlushData(ctx, t, mc, schema.CollectionName)
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// hybrid search
queryVec1 := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector)
queryVec2 := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector)
annReq1 := client.NewAnnRequest(common.DefaultFloatVecFieldName, common.DefaultLimit, queryVec1...).WithSearchParam("ef", "100")
annReq2 := client.NewAnnRequest(common.DefaultFloatVecFieldName, common.DefaultLimit, queryVec2...)
searchRes, errSearch := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReq1, annReq2).WithOutputFields("*"))
common.CheckErr(t, errSearch, true)
common.CheckSearchResult(t, searchRes, common.DefaultNq, common.DefaultLimit)
common.CheckOutputFields(t, []string{common.DefaultInt64FieldName, common.DefaultFloatVecFieldName}, searchRes[0].Fields)
// ignore growing
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption().TWithStart(common.DefaultNb).TWithNb(500))
annReq1.WithIgnoreGrowing(true)
annReq2.WithIgnoreGrowing(true)
searchRes, errSearch = mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReq1, annReq2))
common.CheckErr(t, errSearch, true)
common.CheckSearchResult(t, searchRes, common.DefaultNq, common.DefaultLimit)
for _, hits := range searchRes {
for _, id := range hits.IDs.(*column.ColumnInt64).Data() {
require.Less(t, id, int64(common.DefaultNb))
}
}
}
func TestHybridSearchTemplateParam(t *testing.T) {
t.Parallel()
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create -> insert [0, 3000) -> flush -> index -> load
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64MultiVec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption())
prepare.FlushData(ctx, t, mc, schema.CollectionName)
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// hybrid search
queryVec1 := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector)
queryVec2 := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloat16Vector)
int64Value := 100
annReq1 := client.NewAnnRequest(common.DefaultFloatVecFieldName, common.DefaultLimit, queryVec1...).
WithFilter(fmt.Sprintf("%s > {int64Value}", common.DefaultInt64FieldName)).WithTemplateParam("int64Value", int64Value)
annReq2 := client.NewAnnRequest(common.DefaultFloat16VecFieldName, common.DefaultLimit, queryVec2...).
WithFilter(fmt.Sprintf("%s > {int64Value}", common.DefaultInt64FieldName)).WithTemplateParam("int64Value", 200)
searchRes, errSearch := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReq1, annReq2))
common.CheckErr(t, errSearch, true)
common.CheckSearchResult(t, searchRes, common.DefaultNq, common.DefaultLimit)
for _, hits := range searchRes {
for _, id := range hits.IDs.(*column.ColumnInt64).Data() {
require.Greater(t, id, int64(int64Value))
}
}
}
// hybrid search default -> verify success
func TestHybridSearchMultiVectorsDefault(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
for _, enableDynamic := range []bool{false, true} {
// create -> insert [0, 3000) -> flush -> index -> load
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.AllFields),
hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(enableDynamic))
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption().TWithNb(common.DefaultNb*3))
prepare.FlushData(ctx, t, mc, schema.CollectionName)
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// hybrid search with different limit
type limitGroup struct {
subLimit1 int
subLimit2 int
limit int
}
// Duplicates when aggregating multiple subquery results
limits := []limitGroup{
{subLimit1: 10, subLimit2: 5, limit: 8}, // actual limit 8
{subLimit1: 10, subLimit2: 5, limit: 15}, // actual limit [10, 15]
{subLimit1: 10, subLimit2: 5, limit: 20}, // actual limit [10, 15]
}
expr := fmt.Sprintf("%s > 5", common.DefaultInt64FieldName)
var allFieldsName []string
for _, field := range schema.Fields {
allFieldsName = append(allFieldsName, field.Name)
}
if enableDynamic {
allFieldsName = append(allFieldsName, common.DefaultDynamicFieldName)
}
ch := make(chan struct{}, 3)
wg := sync.WaitGroup{}
testFunc := func(reranker client.Reranker) {
defer func() {
wg.Done()
<-ch
}()
queryVec1 := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector)
queryVec2 := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloat16Vector)
for _, limit := range limits {
// hybrid search
annReq1 := client.NewAnnRequest(common.DefaultFloatVecFieldName, limit.subLimit1, queryVec1...).WithFilter(expr)
annReq2 := client.NewAnnRequest(common.DefaultFloat16VecFieldName, limit.subLimit2, queryVec2...).WithFilter(expr)
searchRes, errSearch := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, limit.limit, annReq1, annReq2).
WithReranker(reranker).WithOutputFields("*"))
common.CheckErr(t, errSearch, true)
actualLimitRange := make([]int, 2)
actualLimitRange[0] = min(max(limit.subLimit1, limit.subLimit2), limit.limit)
actualLimitRange[1] = min(limit.subLimit1+limit.subLimit2, limit.limit)
require.Len(t, searchRes, common.DefaultNq)
common.CheckOutputFields(t, allFieldsName, searchRes[0].Fields)
for _, res := range searchRes {
require.GreaterOrEqual(t, res.ResultCount, actualLimitRange[0])
require.LessOrEqual(t, res.ResultCount, actualLimitRange[1])
require.GreaterOrEqual(t, searchRes[0].IDs.Len(), actualLimitRange[0])
}
}
}
// search with different reranker
for _, reranker := range []client.Reranker{
client.NewRRFReranker(),
client.NewWeightedReranker([]float64{0.8, 0.2}),
client.NewWeightedReranker([]float64{0.0, 0.2}),
client.NewWeightedReranker([]float64{0.4, 1.0}),
} {
reranker := reranker
ch <- struct{}{}
wg.Add(1)
go testFunc(reranker)
}
wg.Wait()
}
}
// invalid limit: 0, -1, max+1
// invalid WeightedReranker params
// invalid fieldName: not exist
// invalid metric type: mismatch
func TestHybridSearchInvalidParams(t *testing.T) {
t.Parallel()
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create -> insert -> flush -> index -> load
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64MultiVec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption())
prepare.FlushData(ctx, t, mc, schema.CollectionName)
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// hybrid search with invalid limit
queryVec1 := hp.GenSearchVectors(1, common.DefaultDim, entity.FieldTypeFloatVector)
queryVec2 := hp.GenSearchVectors(1, common.DefaultDim, entity.FieldTypeBinaryVector)
annReq1 := client.NewAnnRequest(common.DefaultFloatVecFieldName, common.DefaultLimit, queryVec1...)
annReq2 := client.NewAnnRequest(common.DefaultBinaryVecFieldName, common.DefaultLimit, queryVec2...)
for _, invalidLimit := range []int{-1, 0, common.MaxTopK + 1} {
// hybrid search with invalid limit
_, err := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, invalidLimit, annReq1))
common.CheckErr(t, err, false, "should be greater than 0", "should be in range [1, 16384]")
// annRequest with invalid limit
annReq2 := client.NewAnnRequest(common.DefaultFloatVecFieldName, invalidLimit, queryVec1...)
_, err = mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, invalidLimit, annReq2))
common.CheckErr(t, err, false, "should be greater than 0", "should be in range [1, 16384]")
}
// hybrid search with invalid WeightedReranker params
for _, invalidRanker := range []client.Reranker{
client.NewWeightedReranker([]float64{-1, 0.2}),
client.NewWeightedReranker([]float64{1.2, 0.2}),
client.NewWeightedReranker([]float64{0.2}),
client.NewWeightedReranker([]float64{0.2, 0.7, 0.5}),
} {
_, errReranker := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReq1, annReq2).WithReranker(invalidRanker))
common.CheckErr(t, errReranker, false, "rank param weight should be in range [0, 1]",
"the length of weights param mismatch with ann search requests")
}
// invalid fieldName: not exist
annReq3 := client.NewAnnRequest("a", common.DefaultLimit, queryVec1...)
_, errField := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReq3))
common.CheckErr(t, errField, false, "failed to get field schema by name: fieldName(a) not found")
// invalid metric type: mismatch
annReq4 := client.NewAnnRequest(common.DefaultFloatVecFieldName, common.DefaultLimit, queryVec1...).WithSearchParam("metric_type", "L2")
_, errMetric := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReq4))
common.CheckErr(t, errMetric, false, "metric type not match: invalid parameter")
}
// vector type mismatch: vectors: float32, queryVec: binary
// vector dim mismatch
func TestHybridSearchInvalidVectors(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64Vec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption().TWithNb(500))
prepare.FlushData(ctx, t, mc, schema.CollectionName)
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// vector dim or type mismatch
for _, invalidVec := range [][]entity.Vector{
hp.GenSearchVectors(2, common.DefaultDim*2, entity.FieldTypeFloatVector), // vector dim mismatch
hp.GenSearchVectors(1, common.DefaultDim, entity.FieldTypeFloat16Vector), // vector type mismatch
} {
annReq := client.NewAnnRequest(common.DefaultFloatVecFieldName, common.DefaultLimit, invalidVec...)
_, err := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReq))
common.CheckErr(t, err, false, "vector dimension mismatch", "vector type must be the same")
}
}
// hybrid search Pagination -> verify success
func TestHybridSearchMultiVectorsPagination(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64MultiVec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption().TWithNb(common.DefaultNb*5))
prepare.FlushData(ctx, t, mc, schema.CollectionName)
// hybrid search with different offset
queryVec1 := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector)
queryVec2 := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloat16Vector)
annReqDef := client.NewAnnRequest(common.DefaultFloatVecFieldName, common.DefaultLimit, queryVec1...)
// offset 0, -1 -> 0
for _, offset := range []int{0, -1} {
searchRes, err := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReqDef).WithOffset(offset))
common.CheckErr(t, err, true)
common.CheckSearchResult(t, searchRes, common.DefaultNq, common.DefaultLimit)
}
// check for invalid offset externally, not internally
annReqOffset := client.NewAnnRequest(common.DefaultFloat16VecFieldName, common.DefaultLimit, queryVec2...).WithOffset(common.MaxTopK + 1)
res, errSearch := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReqDef, annReqOffset))
common.CheckErr(t, errSearch, true)
common.CheckSearchResult(t, res, common.DefaultNq, common.DefaultLimit)
_, errSearch = mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReqDef, annReqOffset).WithOffset(common.MaxTopK+1))
common.CheckErr(t, errSearch, false, "should be gte than 0", "(offset+limit) should be in range [1, 16384]")
// search with different reranker and offset
for _, reranker := range []client.Reranker{
client.NewRRFReranker(),
client.NewWeightedReranker([]float64{0.8, 0.2}),
client.NewWeightedReranker([]float64{0.0, 0.2}),
client.NewWeightedReranker([]float64{0.4, 1.0}),
} {
annReq1 := client.NewAnnRequest(common.DefaultFloatVecFieldName, common.DefaultLimit, queryVec1...)
annReq2 := client.NewAnnRequest(common.DefaultFloat16VecFieldName, common.DefaultLimit, queryVec2...)
// hybrid search
searchRes, errSearch := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReq1, annReq2).WithReranker(reranker))
common.CheckErr(t, errSearch, true)
offsetRes, errSearch := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, 5, annReq1, annReq2).WithReranker(reranker).WithOffset(5))
common.CheckErr(t, errSearch, true)
common.CheckSearchResult(t, searchRes, common.DefaultNq, common.DefaultLimit)
common.CheckSearchResult(t, offsetRes, common.DefaultNq, 5)
for i := 0; i < len(searchRes); i++ {
require.Equal(t, searchRes[i].IDs.(*column.ColumnInt64).Data()[5:], offsetRes[i].IDs.(*column.ColumnInt64).Data())
}
}
}
// hybrid search Pagination -> verify success
func TestHybridSearchMultiVectorsRangeSearch(t *testing.T) {
t.Parallel()
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create -> insert [0, 3000) -> flush -> index -> load
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64MultiVec), hp.TNewFieldsOption(), hp.TNewSchemaOption())
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption().TWithNb(common.DefaultNb*3))
prepare.FlushData(ctx, t, mc, schema.CollectionName)
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// hybrid search
expr := fmt.Sprintf("%s > 4", common.DefaultInt64FieldName)
queryVec1 := hp.GenSearchVectors(1, common.DefaultDim, entity.FieldTypeFloatVector)
queryVec2 := hp.GenSearchVectors(1, common.DefaultDim, entity.FieldTypeFloat16Vector)
// search with different reranker and offset
for _, reranker := range []client.Reranker{
client.NewRRFReranker(),
client.NewWeightedReranker([]float64{0.8, 0.2}),
client.NewWeightedReranker([]float64{0.5, 0.5}),
} {
annReq1 := client.NewAnnRequest(common.DefaultFloatVecFieldName, common.DefaultLimit*2, queryVec1...).WithSearchParam("radius", "20").WithOffset(1).WithFilter(expr)
annReq2 := client.NewAnnRequest(common.DefaultFloat16VecFieldName, common.DefaultLimit, queryVec2...).WithSearchParam("range_filter", "0.01").WithFilter(expr)
// hybrid search
resRange, errSearch := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReq1, annReq2).WithReranker(reranker))
common.CheckErr(t, errSearch, true)
common.CheckSearchResult(t, resRange, 1, common.DefaultLimit)
for _, res := range resRange {
for _, score := range res.Scores {
require.GreaterOrEqual(t, score, float32(0.01))
require.LessOrEqual(t, score, float32(20))
}
}
}
}
func TestHybridSearchSparseVector(t *testing.T) {
t.Parallel()
idxInverted := index.NewSparseInvertedIndex(entity.IP, 0.2)
idxWand := index.NewSparseWANDIndex(entity.IP, 0.3)
for _, idx := range []index.Index{idxInverted, idxWand} {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create -> insert [0, 3000) -> flush -> index -> load
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VarcharSparseVec), hp.TNewFieldsOption(),
hp.TNewSchemaOption().TWithEnableDynamicField(true))
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema).TWithFieldIndex(map[string]index.Index{common.DefaultSparseVecFieldName: idx}))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption().TWithNb(common.DefaultNb*3))
prepare.FlushData(ctx, t, mc, schema.CollectionName)
// search
queryVec1 := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim*2, entity.FieldTypeSparseVector)
queryVec2 := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeSparseVector)
expr := fmt.Sprintf("%s > 1", common.DefaultInt64FieldName)
for _, reranker := range []client.Reranker{
client.NewRRFReranker(),
client.NewWeightedReranker([]float64{0.5, 0.6}),
} {
// hybrid search
annReq1 := client.NewAnnRequest(common.DefaultSparseVecFieldName, common.DefaultLimit, queryVec1...).WithFilter(expr)
annReq2 := client.NewAnnRequest(common.DefaultSparseVecFieldName, common.DefaultLimit, queryVec2...)
searchRes, errSearch := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReq1, annReq2).
WithReranker(reranker).WithOutputFields("*"))
common.CheckErr(t, errSearch, true)
common.CheckSearchResult(t, searchRes, common.DefaultNq, common.DefaultLimit)
common.CheckErr(t, errSearch, true)
outputFields := []string{common.DefaultInt64FieldName, common.DefaultVarcharFieldName, common.DefaultSparseVecFieldName, common.DefaultDynamicFieldName}
common.CheckOutputFields(t, outputFields, searchRes[0].Fields)
}
}
}
func TestHybridSearchGroupBy(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
// create collection
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.AllFields), hp.TNewFieldsOption(), hp.TNewSchemaOption())
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
ch := make(chan struct{}, 5)
wg := sync.WaitGroup{}
testFunc := func() {
defer func() {
wg.Done()
<-ch
}()
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption().TWithNb(1000))
}
for i := 0; i < 10; i++ {
ch <- struct{}{}
wg.Add(1)
go testFunc()
}
wg.Wait()
prepare.FlushData(ctx, t, mc, schema.CollectionName)
// hybrid search with groupby field
queryVec1 := hp.GenSearchVectors(2, common.DefaultDim, entity.FieldTypeFloatVector)
queryVec2 := hp.GenSearchVectors(2, common.DefaultDim, entity.FieldTypeBFloat16Vector)
annReq1 := client.NewAnnRequest(common.DefaultFloatVecFieldName, common.DefaultLimit, queryVec1...).WithGroupByField(common.DefaultVarcharFieldName)
annReq2 := client.NewAnnRequest(common.DefaultBFloat16VecFieldName, common.DefaultLimit, queryVec2...).WithGroupByField(common.DefaultInt32FieldName)
res, errSearch := mc.HybridSearch(ctx, client.NewHybridSearchOption(schema.CollectionName, common.DefaultLimit, annReq1, annReq2))
common.CheckErr(t, errSearch, true)
common.CheckSearchResult(t, res, 2, common.DefaultLimit)
// TODO hybrid search WithGroupSize, WithStrictGroupSize
}

View File

@ -17,7 +17,6 @@ import (
)
func TestIndexVectorDefault(t *testing.T) {
t.Parallel()
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2)
mc := createDefaultMilvusClient(ctx, t)
@ -50,7 +49,6 @@ func TestIndexVectorDefault(t *testing.T) {
}
func TestIndexVectorIP(t *testing.T) {
t.Parallel()
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2)
mc := createDefaultMilvusClient(ctx, t)
@ -84,7 +82,6 @@ func TestIndexVectorIP(t *testing.T) {
}
func TestIndexVectorCosine(t *testing.T) {
t.Parallel()
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout*2)
mc := createDefaultMilvusClient(ctx, t)
@ -118,7 +115,6 @@ func TestIndexVectorCosine(t *testing.T) {
}
func TestIndexAutoFloatVector(t *testing.T) {
t.Parallel()
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
@ -155,7 +151,6 @@ func TestIndexAutoFloatVector(t *testing.T) {
}
func TestIndexAutoBinaryVector(t *testing.T) {
t.Parallel()
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
@ -196,7 +191,6 @@ func TestIndexAutoBinaryVector(t *testing.T) {
}
func TestIndexAutoSparseVector(t *testing.T) {
t.Parallel()
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
@ -758,7 +752,6 @@ func TestCreateIndexDup(t *testing.T) {
}
func TestCreateIndexSparseVectorGeneric(t *testing.T) {
t.Parallel()
idxInverted := index.NewGenericIndex(common.DefaultSparseVecFieldName, map[string]string{"drop_ratio_build": "0.2", index.MetricTypeKey: "IP", index.IndexTypeKey: "SPARSE_INVERTED_INDEX"})
idxWand := index.NewGenericIndex(common.DefaultSparseVecFieldName, map[string]string{"drop_ratio_build": "0.3", index.MetricTypeKey: "IP", index.IndexTypeKey: "SPARSE_WAND"})
@ -787,7 +780,6 @@ func TestCreateIndexSparseVectorGeneric(t *testing.T) {
}
func TestCreateIndexSparseVector(t *testing.T) {
t.Parallel()
idxInverted1 := index.NewSparseInvertedIndex(entity.IP, 0.2)
idxWand1 := index.NewSparseWANDIndex(entity.IP, 0.3)
for _, idx := range []index.Index{idxInverted1, idxWand1} {

View File

@ -2,6 +2,7 @@ package testcases
import (
"math"
"strconv"
"testing"
"time"
@ -126,6 +127,11 @@ func TestInsertAllFieldsData(t *testing.T) {
flushTak, _ := mc.Flush(ctx, client.NewFlushOption(schema.CollectionName))
err := flushTak.Await(ctx)
common.CheckErr(t, err, true)
// check collection stats
stats, err := mc.GetCollectionStats(ctx, client.NewGetCollectionStatsOption(schema.CollectionName))
common.CheckErr(t, err, true)
require.Equal(t, map[string]string{common.RowCount: strconv.Itoa(common.DefaultNb)}, stats)
}
}
@ -587,6 +593,11 @@ func TestInsertDefaultRows(t *testing.T) {
common.CheckErr(t, errFlush, true)
errFlush = flushTask.Await(ctx)
common.CheckErr(t, errFlush, true)
// check collection stats
stats, err := mc.GetCollectionStats(ctx, client.NewGetCollectionStatsOption(schema.CollectionName))
common.CheckErr(t, err, true)
require.Equal(t, map[string]string{common.RowCount: strconv.Itoa(common.DefaultNb)}, stats)
}
}

View File

@ -33,10 +33,10 @@ func teardown() {
defer mc.Close(ctx)
// clear dbs
dbs, _ := mc.ListDatabases(ctx, clientv2.NewListDatabaseOption())
dbs, _ := mc.ListDatabase(ctx, clientv2.NewListDatabaseOption())
for _, db := range dbs {
if db != common.DefaultDb {
_ = mc.UsingDatabase(ctx, clientv2.NewUseDatabaseOption(db))
_ = mc.UseDatabase(ctx, clientv2.NewUseDatabaseOption(db))
collections, _ := mc.ListCollections(ctx, clientv2.NewListCollectionOption())
for _, coll := range collections {
_ = mc.DropCollection(ctx, clientv2.NewDropCollectionOption(coll))

View File

@ -1,6 +1,7 @@
package testcases
import (
"encoding/json"
"fmt"
"testing"
"time"
@ -604,6 +605,13 @@ func TestQueryCountJsonDynamicExpr(t *testing.T) {
{expr: fmt.Sprintf("%s == [1503, 1504]", common.DefaultJSONFieldName), count: 1}, // json == [1,2]
{expr: fmt.Sprintf("%s[0] > 1", common.DefaultJSONFieldName), count: 1500 / 4}, // json[0] > 1
{expr: fmt.Sprintf("%s[0][0] > 1", common.DefaultJSONFieldName), count: 0}, // json == [1,2]
// Key and value types do not match
{expr: fmt.Sprintf("%s['float'] <= 3000", common.DefaultJSONFieldName), count: common.DefaultNb / 4},
{expr: fmt.Sprintf("%s['float'] <= 3000.0", common.DefaultJSONFieldName), count: common.DefaultNb / 4},
{expr: fmt.Sprintf("%s['string'] > 0", common.DefaultJSONFieldName), count: 0},
{expr: fmt.Sprintf("%s['floatArray'][0] < 1000.0", common.DefaultJSONFieldName), count: 500},
{expr: fmt.Sprintf("%s['stringArray'][0] == '00100'", common.DefaultJSONFieldName), count: 1},
}
for _, _exprCount := range exprCounts {
@ -614,6 +622,50 @@ func TestQueryCountJsonDynamicExpr(t *testing.T) {
}
}
func TestQueryNestedJsonExpr(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VecJSON), hp.TNewFieldsOption(), hp.TNewSchemaOption())
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
pkColumn := hp.GenColumnData(common.DefaultNb, entity.FieldTypeInt64, *hp.TNewDataOption())
vecColumn := hp.GenColumnData(common.DefaultNb, entity.FieldTypeFloatVector, *hp.TNewDataOption())
jsonValues := make([][]byte, 0, common.DefaultNb)
nestedDepth := 100
for i := 0; i < common.DefaultNb; i++ {
var m map[string]interface{}
if i%2 == 0 {
m = make(map[string]interface{})
} else {
m = hp.GenNestedJSON(nestedDepth, i)
}
bs, _ := json.Marshal(&m)
jsonValues = append(jsonValues, bs)
}
jsonColumn := column.NewColumnJSONBytes(common.DefaultJSONFieldName, jsonValues)
_, err := mc.Insert(ctx, client.NewColumnBasedInsertOption(schema.CollectionName, pkColumn, vecColumn, jsonColumn))
common.CheckErr(t, err, true)
type exprCount struct {
expr string
count int64
}
exprKey := hp.GenNestedJSONExprKey(nestedDepth, common.DefaultJSONFieldName)
nestedExpr := exprKey + " < 1000 "
t.Log("https://github.com/milvus-io/milvus/issues/39822")
exprCounts := []exprCount{
//{expr: fmt.Sprintf("json_length(%s) == 0", common.DefaultJSONFieldName), count: common.DefaultNb / 2},
{expr: nestedExpr, count: 500},
}
for _, _exprCount := range exprCounts {
log.Info("TestQueryCountJsonDynamicExpr", zap.String("expr", _exprCount.expr))
countRes, _ := mc.Query(ctx, client.NewQueryOption(schema.CollectionName).WithConsistencyLevel(entity.ClStrong).WithFilter(_exprCount.expr).WithOutputFields(common.QueryCountFieldName))
count, _ := countRes.Fields[0].GetAsInt64(0)
require.Equal(t, _exprCount.count, count)
}
}
// test query with all kinds of array expr
func TestQueryArrayFieldExpr(t *testing.T) {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)

View File

@ -4,6 +4,7 @@ import (
"fmt"
"math/rand"
"strconv"
"sync"
"testing"
"time"
@ -570,32 +571,44 @@ func TestSearchInvalidScannReorderK(t *testing.T) {
// test search with scann index params: with_raw_data and metrics_type [L2, IP, COSINE]
func TestSearchScannAllMetricsWithRawData(t *testing.T) {
t.Parallel()
ch := make(chan struct{}, 3)
wg := sync.WaitGroup{}
testFunc := func(withRawData bool, metricType entity.MetricType) {
defer func() {
wg.Done()
<-ch
}()
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VecJSON),
hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true))
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption())
prepare.FlushData(ctx, t, mc, schema.CollectionName)
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema).TWithFieldIndex(map[string]index.Index{
common.DefaultFloatVecFieldName: index.NewSCANNIndex(metricType, 16, withRawData),
}))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// search and output all fields
vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector)
resSearch, errSearch := mc.Search(ctx, client.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).
WithConsistencyLevel(entity.ClStrong).WithOutputFields("*"))
common.CheckErr(t, errSearch, true)
common.CheckOutputFields(t, []string{
common.DefaultInt64FieldName, common.DefaultJSONFieldName,
common.DefaultFloatVecFieldName, common.DefaultDynamicFieldName,
}, resSearch[0].Fields)
common.CheckSearchResult(t, resSearch, common.DefaultNq, common.DefaultLimit)
}
for _, withRawData := range []bool{true, false} {
for _, metricType := range []entity.MetricType{entity.L2, entity.IP, entity.COSINE} {
ctx := hp.CreateContext(t, time.Second*common.DefaultTimeout)
mc := createDefaultMilvusClient(ctx, t)
prepare, schema := hp.CollPrepare.CreateCollection(ctx, t, mc, hp.NewCreateCollectionParams(hp.Int64VecJSON),
hp.TNewFieldsOption(), hp.TNewSchemaOption().TWithEnableDynamicField(true))
prepare.InsertData(ctx, t, mc, hp.NewInsertParams(schema), hp.TNewDataOption())
prepare.FlushData(ctx, t, mc, schema.CollectionName)
prepare.CreateIndex(ctx, t, mc, hp.TNewIndexParams(schema).TWithFieldIndex(map[string]index.Index{
common.DefaultFloatVecFieldName: index.NewSCANNIndex(metricType, 16, withRawData),
}))
prepare.Load(ctx, t, mc, hp.NewLoadParams(schema.CollectionName))
// search and output all fields
vectors := hp.GenSearchVectors(common.DefaultNq, common.DefaultDim, entity.FieldTypeFloatVector)
resSearch, errSearch := mc.Search(ctx, client.NewSearchOption(schema.CollectionName, common.DefaultLimit, vectors).
WithConsistencyLevel(entity.ClStrong).WithOutputFields("*"))
common.CheckErr(t, errSearch, true)
common.CheckOutputFields(t, []string{
common.DefaultInt64FieldName, common.DefaultJSONFieldName,
common.DefaultFloatVecFieldName, common.DefaultDynamicFieldName,
}, resSearch[0].Fields)
common.CheckSearchResult(t, resSearch, common.DefaultNq, common.DefaultLimit)
ch <- struct{}{}
wg.Add(1)
go testFunc(withRawData, metricType)
}
}
wg.Wait()
}
// test search with valid expression