milvus/internal/datacoord/util_test.go
congqixia 92c0c38e24
fix: validate collection TTL property to prevent compaction stuck (#46717)
If collection TTL property is malformed (e.g., non-numeric value),
compaction tasks would fail silently and get stuck. This change:

- Add centralized GetCollectionTTL/GetCollectionTTLFromMap functions in
pkg/common to handle TTL parsing with proper error handling
- Validate TTL property in createCollectionTask and alterCollectionTask
PreExecute to reject invalid values early
- Refactor datacoord compaction policies to use the new common functions
- Remove duplicated getCollectionTTL from datacoord/util.go

issue: #46716

<!-- This is an auto-generated comment: release notes by coderabbit.ai
-->
- Core invariant: collection.ttl.seconds must be a parseable int64 and
validated at collection creation/alter time so malformed TTLs never
reach compaction/execution codepaths.
- Bug fix (resolves #46716): malformed/non-numeric TTLs could silently
cause compaction tasks to fail/stall; fixed by adding centralized
parsing helpers pkg/common.GetCollectionTTL and GetCollectionTTLFromMap
and validating TTL in createCollectionTask.PreExecute and
alterCollectionTask.PreExecute (calls with default -1 and return
parameter-invalid errors on parse failure).
- Simplification / removed redundancy: eliminated duplicated
getCollectionTTL in internal/datacoord/util.go and replaced ad-hoc TTL
parsing across datacoord (compaction policies, import_util, compaction
triggers) and proxy util with the common helpers, centralizing error
handling and defaulting logic.
- No data loss or behavior regression: valid TTL parsing semantics
unchanged (helpers use identical int64 parsing and default fallback from
paramtable/CommonCfg); validation occurs in PreExecute so existing valid
collections proceed unchanged while malformed values are rejected
early—compaction codepaths now receive only validated TTL values (or
explicit defaults), preventing silent skips without altering valid
execution flows.
<!-- end of auto-generated comment: release notes by coderabbit.ai -->

---------

Signed-off-by: Congqi Xia <congqi.xia@zilliz.com>
2026-01-01 08:13:22 +08:00

179 lines
4.5 KiB
Go

// licensed to the lf ai & data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package datacoord
import (
"context"
"testing"
"time"
"github.com/cockroachdb/errors"
"github.com/stretchr/testify/suite"
"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
"github.com/milvus-io/milvus/pkg/v2/common"
"github.com/milvus-io/milvus/pkg/v2/proto/datapb"
"github.com/milvus-io/milvus/pkg/v2/proto/rootcoordpb"
"github.com/milvus-io/milvus/pkg/v2/util/tsoutil"
)
type UtilSuite struct {
suite.Suite
}
func (suite *UtilSuite) TestVerifyResponse() {
type testCase struct {
resp interface{}
err error
expected error
equalValue bool
}
cases := []testCase{
{
resp: nil,
err: errors.New("boom"),
expected: errors.New("boom"),
equalValue: true,
},
{
resp: nil,
err: nil,
expected: errNilResponse,
equalValue: false,
},
{
resp: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success},
err: nil,
expected: nil,
equalValue: false,
},
{
resp: &commonpb.Status{ErrorCode: commonpb.ErrorCode_UnexpectedError, Reason: "r1"},
err: nil,
expected: errors.New("r1"),
equalValue: true,
},
{
resp: (*commonpb.Status)(nil),
err: nil,
expected: errNilResponse,
equalValue: false,
},
{
resp: &rootcoordpb.AllocIDResponse{
Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_Success},
},
err: nil,
expected: nil,
equalValue: false,
},
{
resp: &rootcoordpb.AllocIDResponse{
Status: &commonpb.Status{ErrorCode: commonpb.ErrorCode_UnexpectedError, Reason: "r2"},
},
err: nil,
expected: errors.New("r2"),
equalValue: true,
},
{
resp: &rootcoordpb.AllocIDResponse{},
err: nil,
expected: errNilStatusResponse,
equalValue: true,
},
{
resp: (*rootcoordpb.AllocIDResponse)(nil),
err: nil,
expected: errNilStatusResponse,
equalValue: true,
},
{
resp: struct{}{},
err: nil,
expected: errUnknownResponseType,
equalValue: false,
},
}
for _, c := range cases {
r := VerifyResponse(c.resp, c.err)
if c.equalValue {
suite.Contains(r.Error(), c.expected.Error())
} else {
suite.Equal(c.expected, r)
}
}
}
func TestUtil(t *testing.T) {
suite.Run(t, new(UtilSuite))
}
type fixedTSOAllocator struct {
fixedTime time.Time
}
func (f *fixedTSOAllocator) AllocTimestamp(_ context.Context) (Timestamp, error) {
return tsoutil.ComposeTS(f.fixedTime.UnixNano()/int64(time.Millisecond), 0), nil
}
func (f *fixedTSOAllocator) AllocID(_ context.Context) (UniqueID, error) {
panic("not implemented") // TODO: Implement
}
func (f *fixedTSOAllocator) AllocN(_ context.Context, _ int64) (UniqueID, UniqueID, error) {
panic("not implemented") // TODO: Implement
}
func (suite *UtilSuite) TestGetZeroTime() {
n := 10
for i := 0; i < n; i++ {
timeGot := getZeroTime()
suite.True(timeGot.IsZero())
}
}
func (suite *UtilSuite) TestGetCollectionAutoCompactionEnabled() {
properties := map[string]string{
common.CollectionAutoCompactionKey: "true",
}
enabled, err := getCollectionAutoCompactionEnabled(properties)
suite.NoError(err)
suite.True(enabled)
properties = map[string]string{
common.CollectionAutoCompactionKey: "bad_value",
}
_, err = getCollectionAutoCompactionEnabled(properties)
suite.Error(err)
enabled, err = getCollectionAutoCompactionEnabled(map[string]string{})
suite.NoError(err)
suite.Equal(Params.DataCoordCfg.EnableAutoCompaction.GetAsBool(), enabled)
}
func (suite *UtilSuite) TestCalculateL0SegmentSize() {
logsize := int64(100)
fields := []*datapb.FieldBinlog{{
FieldID: 102,
Binlogs: []*datapb.Binlog{{LogSize: logsize, MemorySize: logsize}},
}}
suite.Equal(calculateL0SegmentSize(fields), float64(logsize))
}