admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2026-02-02 01:06:41 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: patching search cache param when index meta does not hold one (#30116)

Browse Source 
patch search cache param from index configs when index meta could not
get the search cache size key

issue: #30113 
pr: #30119

Signed-off-by: xianliang <xianliang.li@zilliz.com>
This commit is contained in:
foxspy 2024-01-19 11:50:56 +08:00 committed by GitHub
parent be1470a654
commit 0700434c58
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
5 changed files with 151 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
internal/core/thirdparty/knowhere/CMakeLists.txt vendored
View File

@ -15,7 +15,7 @@
set( KNOWHERE_VERSION v2.2.3 )
set( GIT_REPOSITORY "https://github.com/zilliztech/knowhere.git")
if ( INDEX_ENGINE STREQUAL "cardinal" )
set( KNOWHERE_VERSION 2.2.3 )
set( KNOWHERE_VERSION 2.2.3-hotfix )
set( GIT_REPOSITORY "https://github.com/zilliztech/knowhere-cloud.git")
endif()
message(STATUS "Knowhere repo: ${GIT_REPOSITORY}")

9
internal/datacoord/index_builder.go
View File

@ -30,6 +30,7 @@ import (
"github.com/milvus-io/milvus/internal/types"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/log"
"github.com/milvus-io/milvus/pkg/util/indexparams"
"github.com/milvus-io/milvus/pkg/util/merr"
)
@ -299,6 +300,14 @@ func (ib *indexBuilder) process(buildID UniqueID) bool {
RequestTimeoutMs: Params.MinioCfg.RequestTimeoutMs.GetAsInt64(),
}
}
if isDiskANNIndex(getIndexType(indexParams)) {
var err error
indexParams, err = indexparams.AppendDiskIndexBuildParams(Params, indexParams)
if err != nil {
log.Ctx(ib.ctx).Warn("failed to append index build params", zap.Int64("buildID", buildID),
zap.Int64("nodeID", nodeID), zap.Error(err))
}
}
req := &indexpb.CreateJobRequest{
ClusterID: Params.CommonCfg.ClusterPrefix.GetValue(),
IndexFilePrefix: path.Join(ib.chunkManager.RootPath(), common.SegmentIndexPath),

4
internal/datacoord/util.go
View File

@ -182,6 +182,10 @@ func isFlatIndex(indexType string) bool {
return indexType == flatIndex || indexType == binFlatIndex
}
func isDiskANNIndex(indexType string) bool {
return indexType == diskAnnIndex
}
func parseBuildIDFromFilePath(key string) (UniqueID, error) {
ss := strings.Split(key, "/")
if strings.HasSuffix(key, "/") {

42
pkg/util/indexparams/index_params.go
View File

@ -22,6 +22,7 @@ import (
"strconv"
"unsafe"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/funcutil"
"github.com/milvus-io/milvus/pkg/util/hardware"
@ -187,6 +188,47 @@ func FillDiskIndexParams(params *paramtable.ComponentParam, indexParams map[stri
return nil
}
func GetIndexParams(indexParams []*commonpb.KeyValuePair, key string) string {
for _, param := range indexParams {
if param.Key == key {
return param.Value
}
}
return ""
}
// AppendDiskIndexBuildParams append index params for `buildIndex` (params not exist in `CreateIndex`)
func AppendDiskIndexBuildParams(params *paramtable.ComponentParam, indexParams []*commonpb.KeyValuePair) ([]*commonpb.KeyValuePair, error) {
existedVal := GetIndexParams(indexParams, SearchCacheBudgetRatioKey)
if len(existedVal) > 0 {
return indexParams, nil
}
var searchCacheBudgetGBRatio string
if params.AutoIndexConfig.Enable.GetAsBool() {
extraParams, err := NewBigDataExtraParamsFromJSON(params.AutoIndexConfig.ExtraParams.GetValue())
if err != nil {
return indexParams, fmt.Errorf("index param search_cache_budget_gb_ratio not exist in AutoIndex Config")
}
searchCacheBudgetGBRatio = fmt.Sprintf("%f", extraParams.SearchCacheBudgetGBRatio)
} else {
paramVal, err := strconv.ParseFloat(params.CommonCfg.SearchCacheBudgetGBRatio.GetValue(), 64)
if err != nil {
return indexParams, fmt.Errorf("index param search_cache_budget_gb_ratio not exist in Config")
}
searchCacheBudgetGBRatio = fmt.Sprintf("%f", paramVal)
}
if len(searchCacheBudgetGBRatio) > 0 {
indexParams = append(indexParams,
&commonpb.KeyValuePair{
Key: SearchCacheBudgetRatioKey,
Value: searchCacheBudgetGBRatio,
})
}
return indexParams, nil
}
// SetDiskIndexBuildParams set index build params with ratio params on indexNode
// IndexNode cal build param with ratio params and cpu count, memory count...
func SetDiskIndexBuildParams(indexParams map[string]string, fieldDataSize int64) error {

95
pkg/util/indexparams/index_params_test.go
View File

@ -24,6 +24,7 @@ import (
"github.com/stretchr/testify/assert"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus/pkg/common"
"github.com/milvus-io/milvus/pkg/util/hardware"
"github.com/milvus-io/milvus/pkg/util/paramtable"
@ -124,6 +125,100 @@ func TestDiskIndexParams(t *testing.T) {
assert.Error(t, err)
})
t.Run("patch index build params", func(t *testing.T) {
var params paramtable.ComponentParam
params.Init(paramtable.NewBaseTable(paramtable.SkipRemote(true)))
indexParams := make([]*commonpb.KeyValuePair, 0, 3)
indexParams = append(indexParams,
&commonpb.KeyValuePair{
Key: PQCodeBudgetRatioKey,
Value: "0.125",
})
indexParams = append(indexParams,
&commonpb.KeyValuePair{
Key: NumBuildThreadRatioKey,
Value: "1.0",
})
indexParams = append(indexParams,
&commonpb.KeyValuePair{
Key: BeamWidthRatioKey,
Value: "4.0",
})
indexParams, err := AppendDiskIndexBuildParams(&params, indexParams)
assert.NoError(t, err)
assert.True(t, len(indexParams) == 4)
val := GetIndexParams(indexParams, SearchCacheBudgetRatioKey)
cfgVal, cfgErr := strconv.ParseFloat(params.CommonCfg.SearchCacheBudgetGBRatio.GetValue(), 64)
assert.NoError(t, cfgErr)
iVal, iErr := strconv.ParseFloat(val, 64)
assert.NoError(t, iErr)
assert.Equal(t, cfgVal, iVal)
params.Save(params.AutoIndexConfig.Enable.Key, "true")
jsonStr := `
{
"build_ratio": "{\"pq_code_budget_gb\": 0.125, \"num_threads\": 1}",
"prepare_ratio": "{\"search_cache_budget_gb\": 0.225, \"num_threads\": 8}",
"beamwidth_ratio": "8.0"
}
`
params.Save(params.AutoIndexConfig.ExtraParams.Key, jsonStr)
autoParams := make([]*commonpb.KeyValuePair, 0, 3)
autoParams = append(autoParams,
&commonpb.KeyValuePair{
Key: PQCodeBudgetRatioKey,
Value: "0.125",
})
autoParams = append(autoParams,
&commonpb.KeyValuePair{
Key: NumBuildThreadRatioKey,
Value: "1.0",
})
autoParams = append(autoParams,
&commonpb.KeyValuePair{
Key: BeamWidthRatioKey,
Value: "4.0",
})
autoParams, err = AppendDiskIndexBuildParams(&params, autoParams)
assert.NoError(t, err)
assert.True(t, len(autoParams) == 4)
val = GetIndexParams(autoParams, SearchCacheBudgetRatioKey)
iVal, iErr = strconv.ParseFloat(val, 64)
assert.NoError(t, iErr)
assert.Equal(t, 0.225, iVal)
newJSONStr := `
{
"build_ratio": "{\"pq_code_budget_gb\": 0.125, \"num_threads\": 1}",
"prepare_ratio": "{\"search_cache_budget_gb\": 0.325, \"num_threads\": 8}",
"beamwidth_ratio": "8.0"
}
`
params.Save(params.AutoIndexConfig.ExtraParams.Key, newJSONStr)
autoParams, err = AppendDiskIndexBuildParams(&params, autoParams)
assert.NoError(t, err)
assert.True(t, len(autoParams) == 4)
val = GetIndexParams(autoParams, SearchCacheBudgetRatioKey)
iVal, iErr = strconv.ParseFloat(val, 64)
assert.NoError(t, iErr)
assert.Equal(t, 0.225, iVal)
})
t.Run("set disk index build params", func(t *testing.T) {
indexParams := make(map[string]string)
indexParams[PQCodeBudgetRatioKey] = "0.125"