From 9ba0c4e501ad8ceb1198718bf0c31b53f8a14b89 Mon Sep 17 00:00:00 2001 From: zhagnlu Date: Wed, 24 Dec 2025 19:17:18 +0800 Subject: [PATCH] fix:add json stats version because previous change #46130 (#46467) #42533 Signed-off-by: luzhang Co-authored-by: luzhang --- internal/core/src/common/Consts.h | 3 ++- internal/datanode/index/task_stats.go | 4 +++- internal/querynodev2/segments/segment.go | 6 ++++-- pkg/common/common.go | 3 ++- 4 files changed, 11 insertions(+), 5 deletions(-) diff --git a/internal/core/src/common/Consts.h b/internal/core/src/common/Consts.h index db9970c94c..ac6b8eafaf 100644 --- a/internal/core/src/common/Consts.h +++ b/internal/core/src/common/Consts.h @@ -56,7 +56,8 @@ const char HINTS[] = "hints"; const char JSON_KEY_INDEX_LOG_ROOT_PATH[] = "json_key_index_log"; const char NGRAM_LOG_ROOT_PATH[] = "ngram_log"; constexpr const char* JSON_STATS_ROOT_PATH = "json_stats"; -constexpr const char* JSON_STATS_DATA_FORMAT_VERSION = "2"; +// Version 3: metadata moved to separate meta.json file (instead of parquet metadata) +constexpr const char* JSON_STATS_DATA_FORMAT_VERSION = "3"; constexpr const char* JSON_STATS_SHARED_INDEX_PATH = "shared_key_index"; constexpr const char* JSON_STATS_SHREDDING_DATA_PATH = "shredding_data"; constexpr const char* JSON_STATS_META_FILE_NAME = "meta.json"; diff --git a/internal/datanode/index/task_stats.go b/internal/datanode/index/task_stats.go index 94dbca895e..78a24f84c7 100644 --- a/internal/datanode/index/task_stats.go +++ b/internal/datanode/index/task_stats.go @@ -569,9 +569,11 @@ func (st *statsTask) createJSONKeyStats(ctx context.Context, ) if jsonKeyStatsDataFormat != common.JSONStatsDataFormatVersion { - log.Info("create json key index failed dataformat invalid") + log.Warn("create json key index failed dataformat invalid", zap.Int64("dataformat version", jsonKeyStatsDataFormat), + zap.Int64("code version", common.JSONStatsDataFormatVersion)) return nil } + fieldBinlogs := lo.GroupBy(insertBinlogs, func(binlog *datapb.FieldBinlog) int64 { return binlog.GetFieldID() }) diff --git a/internal/querynodev2/segments/segment.go b/internal/querynodev2/segments/segment.go index e52fe372c0..8e4562576e 100644 --- a/internal/querynodev2/segments/segment.go +++ b/internal/querynodev2/segments/segment.go @@ -1227,13 +1227,15 @@ func (s *LocalSegment) LoadJSONKeyIndex(ctx context.Context, jsonKeyStats *datap return nil } + // for compatibility, we only support load data format version equal to the current data format version + // if the data format version is less than the current version, wait for trigger a stats task again if jsonKeyStats.GetJsonKeyStatsDataFormat() != common.JSONStatsDataFormatVersion { - log.Ctx(ctx).Info("load json key index failed dataformat invalid", zap.Int64("dataformat", jsonKeyStats.GetJsonKeyStatsDataFormat()), zap.Int64("field id", jsonKeyStats.GetFieldID()), zap.Any("json key logs", jsonKeyStats)) + log.Ctx(ctx).Warn("load json key index failed dataformat invalid", zap.Int64("dataformat", jsonKeyStats.GetJsonKeyStatsDataFormat()), zap.Int64("field id", jsonKeyStats.GetFieldID()), zap.Any("json key logs", jsonKeyStats)) return nil } log.Ctx(ctx).Info("load json key index", zap.Int64("field id", jsonKeyStats.GetFieldID()), zap.Any("json key logs", jsonKeyStats)) - if info, ok := s.fieldJSONStats[jsonKeyStats.GetFieldID()]; ok && info.GetDataFormatVersion() >= common.JSONStatsDataFormatVersion { + if _, ok := s.fieldJSONStats[jsonKeyStats.GetFieldID()]; ok { log.Warn("JsonKeyIndexStats already loaded", zap.Int64("field id", jsonKeyStats.GetFieldID()), zap.Any("json key logs", jsonKeyStats)) return nil } diff --git a/pkg/common/common.go b/pkg/common/common.go index 10c6585cfb..443439775a 100644 --- a/pkg/common/common.go +++ b/pkg/common/common.go @@ -134,7 +134,8 @@ const ( ) const ( - JSONStatsDataFormatVersion = 2 + // Version 3: metadata moved to separate meta.json file (instead of parquet metadata) + JSONStatsDataFormatVersion = 3 ) // Search, Index parameter keys