admin/milvus
1
0
Fork 0
mirror of https://gitee.com/milvus-io/milvus.git synced 2025-12-28 14:35:27 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix:add json stats version because previous change #46130 (#46467)

Browse Source 
#42533

Signed-off-by: luzhang <luzhang@zilliz.com>
Co-authored-by: luzhang <luzhang@zilliz.com>
This commit is contained in:
zhagnlu 2025-12-24 19:17:18 +08:00 committed by GitHub
parent 342ba550bf
commit 9ba0c4e501
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 11 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
internal/core/src/common/Consts.h
View File

@ -56,7 +56,8 @@ const char HINTS[] = "hints";
const char JSON_KEY_INDEX_LOG_ROOT_PATH[] = "json_key_index_log";
const char NGRAM_LOG_ROOT_PATH[] = "ngram_log";
constexpr const char* JSON_STATS_ROOT_PATH = "json_stats";
constexpr const char* JSON_STATS_DATA_FORMAT_VERSION = "2";
// Version 3: metadata moved to separate meta.json file (instead of parquet metadata)
constexpr const char* JSON_STATS_DATA_FORMAT_VERSION = "3";
constexpr const char* JSON_STATS_SHARED_INDEX_PATH = "shared_key_index";
constexpr const char* JSON_STATS_SHREDDING_DATA_PATH = "shredding_data";
constexpr const char* JSON_STATS_META_FILE_NAME = "meta.json";

4
internal/datanode/index/task_stats.go
View File

@ -569,9 +569,11 @@ func (st *statsTask) createJSONKeyStats(ctx context.Context,
)
if jsonKeyStatsDataFormat != common.JSONStatsDataFormatVersion {
log.Info("create json key index failed dataformat invalid")
log.Warn("create json key index failed dataformat invalid", zap.Int64("dataformat version", jsonKeyStatsDataFormat),
zap.Int64("code version", common.JSONStatsDataFormatVersion))
return nil
}
fieldBinlogs := lo.GroupBy(insertBinlogs, func(binlog *datapb.FieldBinlog) int64 {
return binlog.GetFieldID()
})

6
internal/querynodev2/segments/segment.go
View File

@ -1227,13 +1227,15 @@ func (s *LocalSegment) LoadJSONKeyIndex(ctx context.Context, jsonKeyStats *datap
return nil
}
// for compatibility, we only support load data format version equal to the current data format version
// if the data format version is less than the current version, wait for trigger a stats task again
if jsonKeyStats.GetJsonKeyStatsDataFormat() != common.JSONStatsDataFormatVersion {
log.Ctx(ctx).Info("load json key index failed dataformat invalid", zap.Int64("dataformat", jsonKeyStats.GetJsonKeyStatsDataFormat()), zap.Int64("field id", jsonKeyStats.GetFieldID()), zap.Any("json key logs", jsonKeyStats))
log.Ctx(ctx).Warn("load json key index failed dataformat invalid", zap.Int64("dataformat", jsonKeyStats.GetJsonKeyStatsDataFormat()), zap.Int64("field id", jsonKeyStats.GetFieldID()), zap.Any("json key logs", jsonKeyStats))
return nil
}
log.Ctx(ctx).Info("load json key index", zap.Int64("field id", jsonKeyStats.GetFieldID()), zap.Any("json key logs", jsonKeyStats))
if info, ok := s.fieldJSONStats[jsonKeyStats.GetFieldID()]; ok && info.GetDataFormatVersion() >= common.JSONStatsDataFormatVersion {
if _, ok := s.fieldJSONStats[jsonKeyStats.GetFieldID()]; ok {
log.Warn("JsonKeyIndexStats already loaded", zap.Int64("field id", jsonKeyStats.GetFieldID()), zap.Any("json key logs", jsonKeyStats))
return nil
}

3
pkg/common/common.go
View File

@ -134,7 +134,8 @@ const (
)
const (
JSONStatsDataFormatVersion = 2
// Version 3: metadata moved to separate meta.json file (instead of parquet metadata)
JSONStatsDataFormatVersion = 3
)
// Search, Index parameter keys