From a3b8bcb19867ec3203ad96e4a17af75b69a4dad0 Mon Sep 17 00:00:00 2001 From: "zhenshan.cao" Date: Tue, 18 Nov 2025 23:05:42 +0800 Subject: [PATCH] fix: correct default value backfill during AddField (#45634) issue: https://github.com/milvus-io/milvus/issues/44585 Signed-off-by: zhenshan.cao --- go.mod | 2 +- go.sum | 40 -- .../parser/planparserv2/parser_visitor.go | 6 +- internal/proxy/http_req_impl.go | 8 + internal/proxy/impl.go | 1 - internal/proxy/service_provider.go | 7 + internal/proxy/task.go | 5 +- internal/proxy/task_database.go | 5 +- internal/proxy/task_query.go | 3 +- internal/proxy/task_search.go | 3 +- internal/proxy/util.go | 3 +- internal/proxy/validate_util.go | 7 +- internal/rootcoord/create_collection_task.go | 69 +--- ...dl_callbacks_alter_collection_add_field.go | 12 + ...l_callbacks_alter_collection_properties.go | 3 +- .../rootcoord/ddl_callbacks_alter_database.go | 3 +- .../ddl_callbacks_create_collection.go | 4 +- .../ddl_callbacks_create_database.go | 3 +- .../rootcoord/describe_collection_task.go | 5 - internal/rootcoord/root_coord.go | 59 --- internal/util/importutilv2/csv/row_parser.go | 4 +- internal/util/importutilv2/json/row_parser.go | 4 +- .../util/importutilv2/numpy/field_reader.go | 4 +- .../util/importutilv2/parquet/field_reader.go | 6 +- pkg/go.sum | 1 + pkg/util/funcutil/timestamptz.go | 200 --------- pkg/util/timestamptz/timestamptz.go | 385 ++++++++++++++++++ .../timestamptz_test.go | 2 +- 28 files changed, 454 insertions(+), 400 deletions(-) delete mode 100644 pkg/util/funcutil/timestamptz.go create mode 100644 pkg/util/timestamptz/timestamptz.go rename pkg/util/{funcutil => timestamptz}/timestamptz_test.go (99%) diff --git a/go.mod b/go.mod index 5041eff3c5..9e88392ef2 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/gin-gonic/gin v1.9.1 github.com/go-playground/validator/v10 v10.14.0 github.com/gofrs/flock v0.8.1 - github.com/golang/protobuf v1.5.4 + github.com/golang/protobuf v1.5.4 // indirect github.com/google/btree v1.1.2 github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 github.com/klauspost/compress v1.17.9 diff --git a/go.sum b/go.sum index 4b0352c1a5..aac05928fd 100644 --- a/go.sum +++ b/go.sum @@ -217,12 +217,10 @@ github.com/casbin/casbin/v2 v2.44.2 h1:mlWtgbX872r707frOq+REaHzfvsl+qQw0Eq+ekzJ7 github.com/casbin/casbin/v2 v2.44.2/go.mod h1:vByNa/Fchek0KZUgG5wEsl7iFsiviAYKRtgrQfcJqHg= github.com/casbin/json-adapter/v2 v2.0.0 h1:nOCN3TK1CJKSNQQ/MnakbU9/cUcNR3N0AxBDnEBLSDI= github.com/casbin/json-adapter/v2 v2.0.0/go.mod h1:LvsfPXXr8CD0ZFucAxawcY9Xb0FtLk3mozJ1qcSTUD4= -github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/certifi/gocertifi v0.0.0-20191021191039-0944d244cd40/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= -github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054/go.mod h1:sGbDF6GwGcLpkNXPUTkMRoywsNa/ol15pxFe6ERfguA= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= @@ -240,7 +238,6 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/cncf/udpa/go v0.0.0-20210930031921-04548b0d99d4/go.mod h1:6pvJx4me5XPnfI9Z40ddWsdw2W/uZgQLFXToKeRcDiI= -github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210805033703-aa0b78936158/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20210922020428-25de7278fc84/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= @@ -328,7 +325,6 @@ github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1m github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= -github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/go-control-plane v0.9.10-0.20210907150352-cf90f659a021/go.mod h1:AFq3mo9L8Lqqiid3OhADV3RfLJnjiw63cSpi+fDTRC0= github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= @@ -458,7 +454,6 @@ github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= -github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= @@ -472,7 +467,6 @@ github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -509,7 +503,6 @@ github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/gomodule/redigo v1.7.1-0.20190724094224-574c33c3df38/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/flatbuffers v24.3.25+incompatible h1:CX395cjN9Kke9mmalRoL3d81AtFUxJM+yDthflgJGkI= @@ -937,7 +930,6 @@ github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3O github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66IdsO+O441Eve7ptJDU= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.20.5 h1:cxppBPuYhUnsO6yo/aoRol4L7q7UFfdm+bR9r+8l63Y= github.com/prometheus/client_golang v1.20.5/go.mod h1:PIEt8X02hGcP8JWbeHyeZ53Y/jReSnHgO035n//V5WE= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -1047,7 +1039,6 @@ github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkU github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= github.com/spf13/cobra v1.1.1/go.mod h1:WnodtKOvamDL/PwE2M4iKs8aMDBZ5Q5klgD3qfVJQMI= -github.com/spf13/cobra v1.1.3/go.mod h1:pGADOWyqRD/YMrPZigI/zbliZ2wVD/23d+is3pSWzOo= github.com/spf13/cobra v1.8.1 h1:e5/vxKd/rZsfSJMUX1agtjeTDf+qv1/JdBF8gg5k9ZM= github.com/spf13/cobra v1.8.1/go.mod h1:wHxEcudfqmLYa8iTfL+OuZPbBZkmvliBWKIezN3kD9Y= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= @@ -1176,45 +1167,29 @@ github.com/zilliztech/woodpecker v0.1.11 h1:XYFIqAk6/zOija3Yu3I1mdHj9YGjX4GNHZdN github.com/zilliztech/woodpecker v0.1.11/go.mod h1:xA7jPkUnnr5S4+LmQghrc/1hNU2kiU8KWZ93Uup2jks= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/bbolt v1.3.12 h1:UAxZAIuJqzFwByP19gZC3zd5robK3FOangrGS+Fdczg= go.etcd.io/bbolt v1.3.12/go.mod h1:Gi2toLZr1jFkuReJm+yEPn7H8wk6ooptePtHYCbCS1g= go.etcd.io/etcd/api/v3 v3.5.0-alpha.0/go.mod h1:mPcW6aZJukV6Aa81LSKpBjQXTWlXB5r74ymPoSWa3Sw= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= -go.etcd.io/etcd/api/v3 v3.5.5 h1:BX4JIbQ7hl7+jL+g+2j5UAr0o1bctCm6/Ct+ArBGkf0= -go.etcd.io/etcd/api/v3 v3.5.5/go.mod h1:KFtNaxGDw4Yx/BA4iPPwevUTAuqcsPxzyX8PHydchN8= go.etcd.io/etcd/api/v3 v3.5.23 h1:tQi/RaO6peOhmf0c11miU3RQIYPZmiL3UzG9V+f8g6k= go.etcd.io/etcd/api/v3 v3.5.23/go.mod h1:QP4ZLWROP49Kk/vPLhudxYQcF4ndhMQ1gvJE4rCTAgc= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= -go.etcd.io/etcd/client/pkg/v3 v3.5.5 h1:9S0JUVvmrVl7wCF39iTQthdaaNIiAaQbmK75ogO6GU8= -go.etcd.io/etcd/client/pkg/v3 v3.5.5/go.mod h1:ggrwbk069qxpKPq8/FKkQ3Xq9y39kbFR4LnKszpRXeQ= go.etcd.io/etcd/client/pkg/v3 v3.5.23 h1:RzwVV28JgOwGl5TUjA47s9IWxl5qQjM2VqSh8wjFFLM= go.etcd.io/etcd/client/pkg/v3 v3.5.23/go.mod h1:IdIjxGUGNy+8HWeVlbBXDqmPqe+n8GsVGVYnccAEZ5o= go.etcd.io/etcd/client/v2 v2.305.0-alpha.0/go.mod h1:kdV+xzCJ3luEBSIeQyB/OEKkWKd8Zkux4sbDeANrosU= go.etcd.io/etcd/client/v2 v2.305.0/go.mod h1:h9puh54ZTgAKtEbut2oe9P4L/oqKCVB6xsXlzd7alYQ= -go.etcd.io/etcd/client/v2 v2.305.5 h1:DktRP60//JJpnPC0VBymAN/7V71GHMdjDCBt4ZPXDjI= -go.etcd.io/etcd/client/v2 v2.305.5/go.mod h1:zQjKllfqfBVyVStbt4FaosoX2iYd8fV/GRy/PbowgP4= go.etcd.io/etcd/client/v2 v2.305.23 h1:lo6nsSHjp3tGsRLrzmM+neVSahXxbhxnfoatEdB6nao= go.etcd.io/etcd/client/v2 v2.305.23/go.mod h1:Up9T9+5M3MMcCj/V0nDfadBERNMxIYf1tdycva1dXM4= go.etcd.io/etcd/client/v3 v3.5.0-alpha.0/go.mod h1:wKt7jgDgf/OfKiYmCq5WFGxOFAkVMLxiiXgLDFhECr8= -go.etcd.io/etcd/client/v3 v3.5.5 h1:q++2WTJbUgpQu4B6hCuT7VkdwaTP7Qz6Daak3WzbrlI= -go.etcd.io/etcd/client/v3 v3.5.5/go.mod h1:aApjR4WGlSumpnJ2kloS75h6aHUmAyaPLjHMxpc7E7c= go.etcd.io/etcd/client/v3 v3.5.23 h1:WN7sypGG326sFP5jLkFqD3anf/k6NNlV5Hy/UxvTPvc= go.etcd.io/etcd/client/v3 v3.5.23/go.mod h1:XTf1oMQi4ZzpXGFoPgvAqbY9JFmTFQqWI1SF4g+hV6o= go.etcd.io/etcd/pkg/v3 v3.5.0-alpha.0/go.mod h1:tV31atvwzcybuqejDoY3oaNRTtlD2l/Ot78Pc9w7DMY= -go.etcd.io/etcd/pkg/v3 v3.5.5 h1:Ablg7T7OkR+AeeeU32kdVhw/AGDsitkKPl7aW73ssjU= -go.etcd.io/etcd/pkg/v3 v3.5.5/go.mod h1:6ksYFxttiUGzC2uxyqiyOEvhAiD0tuIqSZkX3TyPdaE= go.etcd.io/etcd/pkg/v3 v3.5.23 h1:q2skPu7VFC7oerL3MrFqImVACjAFWPlZMjHU0zll9x4= go.etcd.io/etcd/pkg/v3 v3.5.23/go.mod h1:zUWlm5U4HgvDg+0PUWb8TgMF7ZrUiQA8aQq7FxE0WHo= go.etcd.io/etcd/raft/v3 v3.5.0-alpha.0/go.mod h1:FAwse6Zlm5v4tEWZaTjmNhe17Int4Oxbu7+2r0DiD3w= -go.etcd.io/etcd/raft/v3 v3.5.5 h1:Ibz6XyZ60OYyRopu73lLM/P+qco3YtlZMOhnXNS051I= -go.etcd.io/etcd/raft/v3 v3.5.5/go.mod h1:76TA48q03g1y1VpTue92jZLr9lIHKUNcYdZOOGyx8rI= go.etcd.io/etcd/raft/v3 v3.5.23 h1:NhCdh4xz1VsrqHd2c+h6SLvhE95B1Hs7K+ESaAs6LLQ= go.etcd.io/etcd/raft/v3 v3.5.23/go.mod h1:NJz9BGkhGvru47lIc1wL0QHsg5yvHTy6tUpEqM69ERM= go.etcd.io/etcd/server/v3 v3.5.0-alpha.0/go.mod h1:tsKetYpt980ZTpzl/gb+UOJj9RkIyCb1u4wjzMg90BQ= -go.etcd.io/etcd/server/v3 v3.5.5 h1:jNjYm/9s+f9A9r6+SC4RvNaz6AqixpOvhrFdT0PvIj0= -go.etcd.io/etcd/server/v3 v3.5.5/go.mod h1:rZ95vDw/jrvsbj9XpTqPrTAB9/kzchVdhRirySPkUBc= go.etcd.io/etcd/server/v3 v3.5.23 h1:2g1hz32pp1TYMI7xUyifR8gXOlUnTBCfixV+uJ8BqRU= go.etcd.io/etcd/server/v3 v3.5.23/go.mod h1:sZwt/lLSwYQ/S5aAbzSQX2xNw1WA0Fo5noUCVxVYB4g= go.mongodb.org/mongo-driver v1.13.1 h1:YIc7HTYsKndGK4RFzJ3covLz1byri52x0IoMB0Pt/vk= @@ -1230,20 +1205,16 @@ go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJyS go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= go.opentelemetry.io/contrib/detectors/gcp v1.34.0 h1:JRxssobiPg23otYU5SbWtQC//snGVIM3Tx6QRzlQBao= go.opentelemetry.io/contrib/detectors/gcp v1.34.0/go.mod h1:cV4BMFcscUR/ckqLkbfQmF0PRsq8w/lMGzdbCSveBHo= -go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.25.0/go.mod h1:E5NNboN0UqSAki0Atn9kVwaN7I+l25gGxDqBueo/74E= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0 h1:rgMkmiGfix9vFJDcDi1PK8WEQP4FLQwLDfhp5ZLpFeE= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.59.0/go.mod h1:ijPqXp5P6IRRByFVVg9DY8P5HkxkHE5ARIa+86aXPf4= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0 h1:CV7UdSGJt/Ao6Gp4CXckLxVRRsRgDHoI8XjbL3PDl8s= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.59.0/go.mod h1:FRmFuRJfag1IZ2dPkHnEoSFVgTVPUd2qf5Vi69hLb8I= -go.opentelemetry.io/otel v1.0.1/go.mod h1:OPEOD4jIT2SlZPMmwT6FqZz2C0ZNdQqiWcoK6M0SNFU= go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY= go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI= go.opentelemetry.io/otel/exporters/jaeger v1.13.0 h1:VAMoGujbVV8Q0JNM/cEbhzUIWWBxnEqH45HP9iBKN04= go.opentelemetry.io/otel/exporters/jaeger v1.13.0/go.mod h1:fHwbmle6mBFJA1p2ZIhilvffCdq/dM5UTIiCOmEjS+w= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.0.1/go.mod h1:Kv8liBeVNFkkkbilbgWRpV+wWuu+H5xdOT6HAgd30iw= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0 h1:DeFD0VgTZ+Cj6hxravYYZE2W4GlneVH81iAOPjZkzk8= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.20.0/go.mod h1:GijYcYmNpX1KazD5JmWGsi4P7dDTTTnfv1UbGn84MnU= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.0.1/go.mod h1:xOvWoTOrQjxjW61xtOmD/WKGRYb/P4NzRo3bs65U6Rk= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0 h1:gvmNvqrPYovvyRmCSygkUDyL8lC5Tl845MLEwqpxhEU= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.20.0/go.mod h1:vNUq47TGFioo+ffTSnKNdob241vePmtNZnAODKapKd0= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.20.0 h1:CsBiKCiQPdSjS+MlRiqeTI9JDDpSuk0Hb6QTRfwer8k= @@ -1254,16 +1225,13 @@ go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.20.0 h1:4s9HxB4azeeQkhY go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.20.0/go.mod h1:djVA3TUJ2fSdMX0JE5XxFBOaZzprElJoP7fD4vnV2SU= go.opentelemetry.io/otel/metric v1.34.0 h1:+eTR3U0MyfWjRDhmFMxe2SsW64QrZ84AOhvqS7Y+PoQ= go.opentelemetry.io/otel/metric v1.34.0/go.mod h1:CEDrp0fy2D0MvkXE+dPV7cMi8tWZwX3dmaIhwPOaqHE= -go.opentelemetry.io/otel/sdk v1.0.1/go.mod h1:HrdXne+BiwsOHYYkBE5ysIcv2bvdZstxzmCQhxTcZkI= go.opentelemetry.io/otel/sdk v1.34.0 h1:95zS4k/2GOy069d321O8jWgYsW3MzVV+KuSPKp7Wr1A= go.opentelemetry.io/otel/sdk v1.34.0/go.mod h1:0e/pNiaMAqaykJGKbi+tSjWfNNHMTxoC9qANsCzbyxU= go.opentelemetry.io/otel/sdk/metric v1.34.0 h1:5CeK9ujjbFVL5c1PhLuStg1wxA7vQv7ce1EK0Gyvahk= go.opentelemetry.io/otel/sdk/metric v1.34.0/go.mod h1:jQ/r8Ze28zRKoNRdkjCZxfs6YvBTG1+YIqyFVFYec5w= -go.opentelemetry.io/otel/trace v1.0.1/go.mod h1:5g4i4fKLaX2BQpSBsxw8YYcgKpMMSW3x7ZTuYBr3sUk= go.opentelemetry.io/otel/trace v1.34.0 h1:+ouXS2V8Rd4hp4580a8q23bg0azF2nI8cqLYnC8mh/k= go.opentelemetry.io/otel/trace v1.34.0/go.mod h1:Svm7lSjQD7kG7KJ/MUHPVXSDGz2OX4h0M2jHBhmSfRE= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v0.9.0/go.mod h1:1vKfU9rv61e9EVGthD1zNvUbiwPcimSsOPU9brfSHJg= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -1314,7 +1282,6 @@ golang.org/x/crypto v0.0.0-20201012173705-84dcc777aaee/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4= @@ -1501,7 +1468,6 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1566,7 +1532,6 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.10.0 h1:3usCWA8tQn0L8+hFJQNgzpWbd89begxN66o1Ojdn5L4= golang.org/x/time v0.10.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1761,14 +1726,9 @@ google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA5 google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= -google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.41.0/go.mod h1:U3l9uK9J0sini8mHphKoXyaqDA/8VyGnDee1zzIUK6k= google.golang.org/grpc v1.43.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= -google.golang.org/grpc v1.71.0 h1:kF77BGdPTQ4/JZWMlb9VpJ5pa25aqvVqogsxNHHdeBg= -google.golang.org/grpc v1.71.0/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= google.golang.org/grpc v1.71.1 h1:ffsFWr7ygTUscGPI0KKK6TLrGz0476KUvvsbqWK0rPI= google.golang.org/grpc v1.71.1/go.mod h1:H0GRtasmQOh9LkFoCPDu3ZrwUtD1YGE+b2vYBYd/8Ec= google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f h1:rqzndB2lIQGivcXdTuY3Y9NBvr70X+y77woofSRluec= diff --git a/internal/parser/planparserv2/parser_visitor.go b/internal/parser/planparserv2/parser_visitor.go index 577d4acd50..3fee45db03 100644 --- a/internal/parser/planparserv2/parser_visitor.go +++ b/internal/parser/planparserv2/parser_visitor.go @@ -12,7 +12,7 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" parser "github.com/milvus-io/milvus/internal/parser/planparserv2/generated" "github.com/milvus-io/milvus/pkg/v2/proto/planpb" - "github.com/milvus-io/milvus/pkg/v2/util/funcutil" + "github.com/milvus-io/milvus/pkg/v2/util/timestamptz" "github.com/milvus-io/milvus/pkg/v2/util/typeutil" ) @@ -2013,7 +2013,7 @@ func (v *ParserVisitor) VisitTimestamptzCompareForward(ctx *parser.TimestamptzCo compareOp := cmpOpMap[ctx.GetOp2().GetTokenType()] - timestamptzInt64, err := funcutil.ValidateAndReturnUnixMicroTz(unquotedCompareStr, v.args.Timezone) + timestamptzInt64, err := timestamptz.ValidateAndReturnUnixMicroTz(unquotedCompareStr, v.args.Timezone) if err != nil { return err } @@ -2077,7 +2077,7 @@ func (v *ParserVisitor) VisitTimestamptzCompareReverse(ctx *parser.TimestamptzCo return fmt.Errorf("unsupported comparison operator for reverse Timestamptz: %s", ctx.GetOp2().GetText()) } - timestamptzInt64, err := funcutil.ValidateAndReturnUnixMicroTz(unquotedCompareStr, v.args.Timezone) + timestamptzInt64, err := timestamptz.ValidateAndReturnUnixMicroTz(unquotedCompareStr, v.args.Timezone) if err != nil { return err } diff --git a/internal/proxy/http_req_impl.go b/internal/proxy/http_req_impl.go index 96d989a0e9..2ea9d6f6e6 100644 --- a/internal/proxy/http_req_impl.go +++ b/internal/proxy/http_req_impl.go @@ -37,6 +37,7 @@ import ( "github.com/milvus-io/milvus/pkg/v2/util/merr" "github.com/milvus-io/milvus/pkg/v2/util/metricsinfo" "github.com/milvus-io/milvus/pkg/v2/util/paramtable" + "github.com/milvus-io/milvus/pkg/v2/util/timestamptz" "github.com/milvus-io/milvus/pkg/v2/util/typeutil" ) @@ -350,6 +351,13 @@ func describeCollection(node *Proxy) gin.HandlerFunc { return } + // Convert TIMESTAMPTZ default values back to string format for the user. + if err := timestamptz.RewriteTimestampTzDefaultValueToString(describeCollectionResp.Schema); err != nil { + c.AbortWithStatusJSON(http.StatusInternalServerError, gin.H{ + mhttp.HTTPReturnMessage: err.Error(), + }) + } + describePartitionResp, err := rootCoord.ShowPartitions(c, &milvuspb.ShowPartitionsRequest{ Base: &commonpb.MsgBase{ MsgType: commonpb.MsgType_ShowPartitions, diff --git a/internal/proxy/impl.go b/internal/proxy/impl.go index e065030c19..65a855c42d 100644 --- a/internal/proxy/impl.go +++ b/internal/proxy/impl.go @@ -842,7 +842,6 @@ func (node *Proxy) BatchDescribeCollection(ctx context.Context, request *milvusp CollectionName: collectionName, } } - responses = append(responses, describeCollectionResponse) } diff --git a/internal/proxy/service_provider.go b/internal/proxy/service_provider.go index a508ee5886..c5f874f4ee 100644 --- a/internal/proxy/service_provider.go +++ b/internal/proxy/service_provider.go @@ -18,6 +18,7 @@ import ( "github.com/milvus-io/milvus/pkg/v2/util/merr" "github.com/milvus-io/milvus/pkg/v2/util/paramtable" "github.com/milvus-io/milvus/pkg/v2/util/timerecord" + "github.com/milvus-io/milvus/pkg/v2/util/timestamptz" "github.com/milvus-io/milvus/pkg/v2/util/typeutil" ) @@ -207,6 +208,12 @@ func (node *CachedProxyServiceProvider) DescribeCollection(ctx context.Context, return nil, err } + err = timestamptz.RewriteTimestampTzDefaultValueToString(resp.Schema) + if err != nil { + log.Info("failed to rewrite timestamp value", zap.Error(err)) + return nil, err + } + resp.CollectionID = c.collID resp.UpdateTimestamp = c.updateTimestamp resp.UpdateTimestampStr = fmt.Sprintf("%d", c.updateTimestamp) diff --git a/internal/proxy/task.go b/internal/proxy/task.go index 9618cb5d66..60317c0fe9 100644 --- a/internal/proxy/task.go +++ b/internal/proxy/task.go @@ -42,6 +42,7 @@ import ( "github.com/milvus-io/milvus/pkg/v2/util/funcutil" "github.com/milvus-io/milvus/pkg/v2/util/merr" "github.com/milvus-io/milvus/pkg/v2/util/paramtable" + "github.com/milvus-io/milvus/pkg/v2/util/timestamptz" "github.com/milvus-io/milvus/pkg/v2/util/typeutil" ) @@ -423,7 +424,7 @@ func (t *createCollectionTask) PreExecute(ctx context.Context) error { // Validate timezone tz, exist := funcutil.TryGetAttrByKeyFromRepeatedKV(common.TimezoneKey, t.GetProperties()) - if exist && !funcutil.IsTimezoneValid(tz) { + if exist && !timestamptz.IsTimezoneValid(tz) { return merr.WrapErrParameterInvalidMsg("unknown or invalid IANA Time Zone ID: %s", tz) } @@ -1214,7 +1215,7 @@ func (t *alterCollectionTask) PreExecute(ctx context.Context) error { } // Check the validation of timezone userDefinedTimezone, exist := funcutil.TryGetAttrByKeyFromRepeatedKV(common.TimezoneKey, t.Properties) - if exist && !funcutil.IsTimezoneValid(userDefinedTimezone) { + if exist && !timestamptz.IsTimezoneValid(userDefinedTimezone) { return merr.WrapErrParameterInvalidMsg("unknown or invalid IANA Time Zone ID: %s", userDefinedTimezone) } } else if len(t.GetDeleteKeys()) > 0 { diff --git a/internal/proxy/task_database.go b/internal/proxy/task_database.go index 93730570be..da433e22b6 100644 --- a/internal/proxy/task_database.go +++ b/internal/proxy/task_database.go @@ -16,6 +16,7 @@ import ( "github.com/milvus-io/milvus/pkg/v2/util/funcutil" "github.com/milvus-io/milvus/pkg/v2/util/merr" "github.com/milvus-io/milvus/pkg/v2/util/paramtable" + "github.com/milvus-io/milvus/pkg/v2/util/timestamptz" ) type createDatabaseTask struct { @@ -74,7 +75,7 @@ func (cdt *createDatabaseTask) PreExecute(ctx context.Context) error { return err } tz, exist := funcutil.TryGetAttrByKeyFromRepeatedKV(common.TimezoneKey, cdt.GetProperties()) - if exist && !funcutil.IsTimezoneValid(tz) { + if exist && !timestamptz.IsTimezoneValid(tz) { return merr.WrapErrParameterInvalidMsg("unknown or invalid IANA Time Zone ID: %s", tz) } return nil @@ -277,7 +278,7 @@ func (t *alterDatabaseTask) PreExecute(ctx context.Context) error { if len(t.GetProperties()) > 0 { // Check the validation of timezone userDefinedTimezone, exist := funcutil.TryGetAttrByKeyFromRepeatedKV(common.TimezoneKey, t.Properties) - if exist && !funcutil.IsTimezoneValid(userDefinedTimezone) { + if exist && !timestamptz.IsTimezoneValid(userDefinedTimezone) { return merr.WrapErrParameterInvalidMsg("unknown or invalid IANA Time Zone ID: %s", userDefinedTimezone) } } diff --git a/internal/proxy/task_query.go b/internal/proxy/task_query.go index 4e625ace17..d3ed5b6548 100644 --- a/internal/proxy/task_query.go +++ b/internal/proxy/task_query.go @@ -34,6 +34,7 @@ import ( "github.com/milvus-io/milvus/pkg/v2/util/merr" "github.com/milvus-io/milvus/pkg/v2/util/paramtable" "github.com/milvus-io/milvus/pkg/v2/util/timerecord" + "github.com/milvus-io/milvus/pkg/v2/util/timestamptz" "github.com/milvus-io/milvus/pkg/v2/util/tsoutil" "github.com/milvus-io/milvus/pkg/v2/util/typeutil" ) @@ -242,7 +243,7 @@ func parseQueryParams(queryParamsPair []*commonpb.KeyValuePair) (*queryParams, e } timezone, _ = funcutil.TryGetAttrByKeyFromRepeatedKV(common.TimezoneKey, queryParamsPair) - if (timezone != "") && !funcutil.IsTimezoneValid(timezone) { + if (timezone != "") && !timestamptz.IsTimezoneValid(timezone) { return nil, merr.WrapErrParameterInvalidMsg("unknown or invalid IANA Time Zone ID: %s", timezone) } diff --git a/internal/proxy/task_search.go b/internal/proxy/task_search.go index 0f61cd3312..e4a901d59c 100644 --- a/internal/proxy/task_search.go +++ b/internal/proxy/task_search.go @@ -39,6 +39,7 @@ import ( "github.com/milvus-io/milvus/pkg/v2/util/metric" "github.com/milvus-io/milvus/pkg/v2/util/paramtable" "github.com/milvus-io/milvus/pkg/v2/util/timerecord" + "github.com/milvus-io/milvus/pkg/v2/util/timestamptz" "github.com/milvus-io/milvus/pkg/v2/util/tsoutil" "github.com/milvus-io/milvus/pkg/v2/util/typeutil" ) @@ -292,7 +293,7 @@ func (t *searchTask) PreExecute(ctx context.Context) error { timezone, exist := funcutil.TryGetAttrByKeyFromRepeatedKV(common.TimezoneKey, t.request.SearchParams) if exist { - if !funcutil.IsTimezoneValid(timezone) { + if !timestamptz.IsTimezoneValid(timezone) { log.Info("get invalid timezone from request", zap.String("timezone", timezone)) return merr.WrapErrParameterInvalidMsg("unknown or invalid IANA Time Zone ID: %s", timezone) } diff --git a/internal/proxy/util.go b/internal/proxy/util.go index ae0698dec9..aec94e1427 100644 --- a/internal/proxy/util.go +++ b/internal/proxy/util.go @@ -60,6 +60,7 @@ import ( "github.com/milvus-io/milvus/pkg/v2/util/merr" "github.com/milvus-io/milvus/pkg/v2/util/metric" "github.com/milvus-io/milvus/pkg/v2/util/paramtable" + "github.com/milvus-io/milvus/pkg/v2/util/timestamptz" "github.com/milvus-io/milvus/pkg/v2/util/tsoutil" "github.com/milvus-io/milvus/pkg/v2/util/typeutil" ) @@ -2901,7 +2902,7 @@ func timestamptzUTC2IsoStr(results []*schemapb.FieldData, colTimezone string) er localTime := t.In(location) // 3. Format using the optimized logic (max 6 digits, no trailing zeros) - isoStrings[i] = funcutil.FormatTimeMicroWithoutTrailingZeros(localTime) + isoStrings[i] = timestamptz.FormatTimeMicroWithoutTrailingZeros(localTime) } // Replace the TimestamptzData with the new StringData in place. diff --git a/internal/proxy/validate_util.go b/internal/proxy/validate_util.go index 917532aecb..e52e8c548f 100644 --- a/internal/proxy/validate_util.go +++ b/internal/proxy/validate_util.go @@ -19,6 +19,7 @@ import ( "github.com/milvus-io/milvus/pkg/v2/util/merr" "github.com/milvus-io/milvus/pkg/v2/util/parameterutil" "github.com/milvus-io/milvus/pkg/v2/util/paramtable" + "github.com/milvus-io/milvus/pkg/v2/util/timestamptz" "github.com/milvus-io/milvus/pkg/v2/util/typeutil" ) @@ -590,7 +591,7 @@ func FillWithDefaultValue(field *schemapb.FieldData, fieldSchema *schemapb.Field // as UTC/the collection's primary timezone, the 'common.DefaultTimezone' passed here // as the fallback timezone is generally inconsequential (negligible) // for the final conversion result in this specific context. - defaultValue, _ = funcutil.ValidateAndReturnUnixMicroTz(strDefaultValue, common.DefaultTimezone) + defaultValue, _ = timestamptz.ValidateAndReturnUnixMicroTz(strDefaultValue, common.DefaultTimezone) } } sd.TimestamptzData.Data, err = fillWithDefaultValueImpl(sd.TimestamptzData.Data, defaultValue, field.GetValidData()) @@ -1165,8 +1166,8 @@ func (v *validateUtil) checkTimestamptzFieldData(field *schemapb.FieldData, time // 2. Validation and Conversion Loop for i, isoStr := range stringData { - // Use the centralized parser (funcutil.ParseTimeTz) for validation and parsing. - t, err := funcutil.ParseTimeTz(isoStr, timezone) + // Use the centralized parser (timestamptz.ParseTimeTz) for validation and parsing. + t, err := timestamptz.ParseTimeTz(isoStr, timezone) if err != nil { log.Warn("cannot parse timestamptz string", zap.String("timestamp_string", isoStr), zap.Error(err)) // Use the recommended refined error message structure diff --git a/internal/rootcoord/create_collection_task.go b/internal/rootcoord/create_collection_task.go index 25243ca8b5..7190efaf21 100644 --- a/internal/rootcoord/create_collection_task.go +++ b/internal/rootcoord/create_collection_task.go @@ -40,6 +40,7 @@ import ( "github.com/milvus-io/milvus/pkg/v2/util/funcutil" "github.com/milvus-io/milvus/pkg/v2/util/merr" "github.com/milvus-io/milvus/pkg/v2/util/paramtable" + "github.com/milvus-io/milvus/pkg/v2/util/timestamptz" "github.com/milvus-io/milvus/pkg/v2/util/typeutil" ) @@ -153,70 +154,6 @@ func checkGeometryDefaultValue(value string) error { return nil } -// checkAndRewriteTimestampTzDefaultValue processes the collection schema to validate -// and rewrite default values for TIMESTAMPTZ fields. -// -// Background: -// 1. TIMESTAMPTZ default values are initially stored as user-provided ISO 8601 strings -// (in ValueField.GetStringData()). -// 2. Milvus stores TIMESTAMPTZ data internally as UTC microseconds (int64). -// -// Logic: -// The function iterates through all fields of type DataType_Timestamptz. For each field -// with a default value: -// 1. It retrieves the collection's default timezone if no offset is present in the string. -// 2. It calls ValidateAndReturnUnixMicroTz to validate the string (including the UTC -// offset range check) and convert it to the absolute UTC microsecond (int64) value. -// 3. It rewrites the ValueField, setting the LongData field with the calculated int64 -// value, thereby replacing the initial string representation. -func checkAndRewriteTimestampTzDefaultValue(schema *schemapb.CollectionSchema) error { - // 1. Get the collection-level default timezone. - // Assuming common.TimezoneKey and common.DefaultTimezone are defined constants. - timezone, exist := funcutil.TryGetAttrByKeyFromRepeatedKV(common.TimezoneKey, schema.GetProperties()) - if !exist { - timezone = common.DefaultTimezone - } - - for _, fieldSchema := range schema.GetFields() { - // Only process TIMESTAMPTZ fields. - if fieldSchema.GetDataType() != schemapb.DataType_Timestamptz { - continue - } - - defaultValue := fieldSchema.GetDefaultValue() - if defaultValue == nil { - continue - } - - // 2. Read the default value as a string (the input format). - // We expect the default value to be set in string_data initially. - stringTz := defaultValue.GetStringData() - if stringTz == "" { - // Skip or handle empty string default values if necessary. - continue - } - - // 3. Validate the string and convert it to UTC microsecond (int64). - // This also performs the critical UTC offset range validation. - utcMicro, err := funcutil.ValidateAndReturnUnixMicroTz(stringTz, timezone) - if err != nil { - // If validation fails (e.g., invalid format or illegal offset), return error immediately. - return err - } - - // 4. Rewrite the default value to store the UTC microsecond (int64). - // By setting ValueField_LongData, the oneof field in the protobuf structure - // automatically switches from string_data to long_data. - defaultValue.Data = &schemapb.ValueField_LongData{ - LongData: utcMicro, - } - - // The original string_data field is now cleared due to the oneof nature, - // and the default value is correctly represented as an int64 microsecond value. - } - return nil -} - func hasSystemFields(schema *schemapb.CollectionSchema, systemFields []string) bool { for _, f := range schema.GetFields() { if funcutil.SliceContain(systemFields, f.GetName()) { @@ -239,7 +176,7 @@ func (t *createCollectionTask) validateSchema(ctx context.Context, schema *schem } // Validate default - if err := checkAndRewriteTimestampTzDefaultValue(schema); err != nil { + if err := timestamptz.CheckAndRewriteTimestampTzDefaultValue(schema); err != nil { return err } @@ -454,7 +391,7 @@ func (t *createCollectionTask) prepareSchema(ctx context.Context) error { // Validate timezone tz, exist := funcutil.TryGetAttrByKeyFromRepeatedKV(common.TimezoneKey, t.Req.GetProperties()) - if exist && !funcutil.IsTimezoneValid(tz) { + if exist && !timestamptz.IsTimezoneValid(tz) { return merr.WrapErrParameterInvalidMsg("unknown or invalid IANA Time Zone ID: %s", tz) } diff --git a/internal/rootcoord/ddl_callbacks_alter_collection_add_field.go b/internal/rootcoord/ddl_callbacks_alter_collection_add_field.go index 8193e9a524..c80d67efb8 100644 --- a/internal/rootcoord/ddl_callbacks_alter_collection_add_field.go +++ b/internal/rootcoord/ddl_callbacks_alter_collection_add_field.go @@ -11,9 +11,12 @@ import ( "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" "github.com/milvus-io/milvus/internal/distributed/streaming" "github.com/milvus-io/milvus/internal/metastore/model" + "github.com/milvus-io/milvus/pkg/v2/common" "github.com/milvus-io/milvus/pkg/v2/proto/messagespb" "github.com/milvus-io/milvus/pkg/v2/streaming/util/message" + "github.com/milvus-io/milvus/pkg/v2/util/funcutil" "github.com/milvus-io/milvus/pkg/v2/util/merr" + "github.com/milvus-io/milvus/pkg/v2/util/timestamptz" "github.com/milvus-io/milvus/pkg/v2/util/typeutil" ) @@ -39,6 +42,15 @@ func (c *Core) broadcastAlterCollectionForAddField(ctx context.Context, req *mil if err := checkFieldSchema([]*schemapb.FieldSchema{fieldSchema}); err != nil { return errors.Wrap(err, "failed to check field schema") } + + if fieldSchema.GetDataType() == schemapb.DataType_Timestamptz { + timezone, exist := funcutil.TryGetAttrByKeyFromRepeatedKV(common.TimezoneKey, coll.Properties) + if !exist { + timezone = common.DefaultTimezone + } + timestamptz.CheckAndRewriteTimestampTzDefaultValueForFieldSchema(fieldSchema, timezone) + } + // check if the field already exists for _, field := range coll.Fields { if field.Name == fieldSchema.Name { diff --git a/internal/rootcoord/ddl_callbacks_alter_collection_properties.go b/internal/rootcoord/ddl_callbacks_alter_collection_properties.go index f0619dd12f..b0edb2ec05 100644 --- a/internal/rootcoord/ddl_callbacks_alter_collection_properties.go +++ b/internal/rootcoord/ddl_callbacks_alter_collection_properties.go @@ -21,6 +21,7 @@ import ( "github.com/milvus-io/milvus/pkg/v2/streaming/util/message/ce" "github.com/milvus-io/milvus/pkg/v2/util/funcutil" "github.com/milvus-io/milvus/pkg/v2/util/merr" + "github.com/milvus-io/milvus/pkg/v2/util/timestamptz" "github.com/milvus-io/milvus/pkg/v2/util/typeutil" ) @@ -48,7 +49,7 @@ func (c *Core) broadcastAlterCollectionForAlterCollection(ctx context.Context, r // Validate timezone tz, exist := funcutil.TryGetAttrByKeyFromRepeatedKV(common.TimezoneKey, req.GetProperties()) - if exist && !funcutil.IsTimezoneValid(tz) { + if exist && !timestamptz.IsTimezoneValid(tz) { return merr.WrapErrParameterInvalidMsg("unknown or invalid IANA Time Zone ID: %s", tz) } diff --git a/internal/rootcoord/ddl_callbacks_alter_database.go b/internal/rootcoord/ddl_callbacks_alter_database.go index 6a3ba1b12d..fa73b97c40 100644 --- a/internal/rootcoord/ddl_callbacks_alter_database.go +++ b/internal/rootcoord/ddl_callbacks_alter_database.go @@ -34,6 +34,7 @@ import ( "github.com/milvus-io/milvus/pkg/v2/streaming/util/message/ce" "github.com/milvus-io/milvus/pkg/v2/util/funcutil" "github.com/milvus-io/milvus/pkg/v2/util/merr" + "github.com/milvus-io/milvus/pkg/v2/util/timestamptz" "github.com/milvus-io/milvus/pkg/v2/util/typeutil" ) @@ -55,7 +56,7 @@ func (c *Core) broadcastAlterDatabase(ctx context.Context, req *rootcoordpb.Alte // Validate timezone tz, exist := funcutil.TryGetAttrByKeyFromRepeatedKV(common.TimezoneKey, req.GetProperties()) - if exist && !funcutil.IsTimezoneValid(tz) { + if exist && !timestamptz.IsTimezoneValid(tz) { return merr.WrapErrParameterInvalidMsg("unknown or invalid IANA Time Zone ID: %s", tz) } diff --git a/internal/rootcoord/ddl_callbacks_create_collection.go b/internal/rootcoord/ddl_callbacks_create_collection.go index ae29af35d8..5a09e15456 100644 --- a/internal/rootcoord/ddl_callbacks_create_collection.go +++ b/internal/rootcoord/ddl_callbacks_create_collection.go @@ -62,7 +62,7 @@ func (c *Core) broadcastCreateCollectionV1(ctx context.Context, req *milvuspb.Cr } defer broadcaster.Close() - // prepare and validate the create collection message. + // prepare and validate the creation collection message. createCollectionTask := createCollectionTask{ Core: c, Req: req, @@ -77,7 +77,7 @@ func (c *Core) broadcastCreateCollectionV1(ctx context.Context, req *milvuspb.Cr return err } - // setup the broadcast virtual channels and control channel, then make a broadcast message. + // set up the broadcast virtual channels and control channel, then make a broadcast message. broadcastChannel := make([]string, 0, createCollectionTask.Req.ShardsNum+1) broadcastChannel = append(broadcastChannel, streaming.WAL().ControlChannel()) for i := 0; i < int(createCollectionTask.Req.ShardsNum); i++ { diff --git a/internal/rootcoord/ddl_callbacks_create_database.go b/internal/rootcoord/ddl_callbacks_create_database.go index c3f7531a10..889bad78d8 100644 --- a/internal/rootcoord/ddl_callbacks_create_database.go +++ b/internal/rootcoord/ddl_callbacks_create_database.go @@ -33,6 +33,7 @@ import ( "github.com/milvus-io/milvus/pkg/v2/streaming/util/message/ce" "github.com/milvus-io/milvus/pkg/v2/util/funcutil" "github.com/milvus-io/milvus/pkg/v2/util/merr" + "github.com/milvus-io/milvus/pkg/v2/util/timestamptz" ) func (c *Core) broadcastCreateDatabase(ctx context.Context, req *milvuspb.CreateDatabaseRequest) error { @@ -58,7 +59,7 @@ func (c *Core) broadcastCreateDatabase(ctx context.Context, req *milvuspb.Create return errors.Wrap(err, "failed to tidy database cipher properties") } tz, exist := funcutil.TryGetAttrByKeyFromRepeatedKV(common.TimezoneKey, properties) - if exist && !funcutil.IsTimezoneValid(tz) { + if exist && !timestamptz.IsTimezoneValid(tz) { return merr.WrapErrParameterInvalidMsg("unknown or invalid IANA Time Zone ID: %s", tz) } msg := message.NewCreateDatabaseMessageBuilderV2(). diff --git a/internal/rootcoord/describe_collection_task.go b/internal/rootcoord/describe_collection_task.go index 630f035e09..d5c6e6d22b 100644 --- a/internal/rootcoord/describe_collection_task.go +++ b/internal/rootcoord/describe_collection_task.go @@ -66,11 +66,6 @@ func (t *describeCollectionTask) Execute(ctx context.Context) (err error) { return err } t.Rsp = convertModelToDesc(coll, aliases, db.Name) - // NEW STEP: Convert TIMESTAMPTZ default values back to string format for the user. - err = rewriteTimestampTzDefaultValueToString(t.Rsp) - if err != nil { - return err - } t.Rsp.RequestTime = t.ts return nil } diff --git a/internal/rootcoord/root_coord.go b/internal/rootcoord/root_coord.go index 6e72d470b2..83b4c05249 100644 --- a/internal/rootcoord/root_coord.go +++ b/internal/rootcoord/root_coord.go @@ -1072,65 +1072,6 @@ func convertModelToDesc(collInfo *model.Collection, aliases []string, dbName str return resp } -// rewriteTimestampTzDefaultValueToString converts the default_value of TIMESTAMPTZ fields -// in the DescribeCollectionResponse from the internal int64 (UTC microsecond) format -// back to a human-readable, timezone-aware string (RFC3339Nano). -// -// This is necessary because TIMESTAMPTZ default values are stored internally as int64 -// after validation but must be returned to the user as a string, respecting the -// collection's default timezone for display purposes if no explicit offset was stored. -func rewriteTimestampTzDefaultValueToString(resp *milvuspb.DescribeCollectionResponse) error { - if resp.GetSchema() == nil { - return nil - } - - // 1. Determine the target timezone for display. - // This is typically stored in the collection properties. - timezone, exist := funcutil.TryGetAttrByKeyFromRepeatedKV(common.TimezoneKey, resp.GetSchema().GetProperties()) - if !exist { - timezone = common.DefaultTimezone // Fallback to a default, like "UTC" - } - - // 2. Iterate through all fields in the schema. - for _, fieldSchema := range resp.Schema.GetFields() { - // Only process TIMESTAMPTZ fields. - if fieldSchema.GetDataType() != schemapb.DataType_Timestamptz { - continue - } - - defaultValue := fieldSchema.GetDefaultValue() - if defaultValue == nil { - continue - } - - // 3. Check if the default value is stored in the internal int64 (LongData) format. - // If it's not LongData, we assume it's either unset or already a string (which shouldn't happen - // if the creation flow worked correctly). - utcMicro, ok := defaultValue.GetData().(*schemapb.ValueField_LongData) - if !ok { - continue // Skip if not stored as LongData (int64) - } - - ts := utcMicro.LongData - - // 4. Convert the int64 microsecond value back to a timezone-aware string. - tzString, err := funcutil.ConvertUnixMicroToTimezoneString(ts, timezone) - if err != nil { - // In a real system, you might log the error and use the raw int64 as a fallback string, - // but here we'll set a placeholder string to avoid crashing. - tzString = fmt.Sprintf("Error converting timestamp: %v", err) - return errors.Wrap(err, tzString) - } - - // 5. Rewrite the default value field in the response schema. - // The protobuf oneof structure ensures setting one field clears the others. - fieldSchema.GetDefaultValue().Data = &schemapb.ValueField_StringData{ - StringData: tzString, - } - } - return nil -} - func (c *Core) describeCollectionImpl(ctx context.Context, in *milvuspb.DescribeCollectionRequest, allowUnavailable bool) (*milvuspb.DescribeCollectionResponse, error) { if err := merr.CheckHealthy(c.GetStateCode()); err != nil { return &milvuspb.DescribeCollectionResponse{ diff --git a/internal/util/importutilv2/csv/row_parser.go b/internal/util/importutilv2/csv/row_parser.go index 79b58527eb..b5f81e4454 100644 --- a/internal/util/importutilv2/csv/row_parser.go +++ b/internal/util/importutilv2/csv/row_parser.go @@ -28,9 +28,9 @@ import ( "github.com/milvus-io/milvus/internal/util/importutilv2/common" "github.com/milvus-io/milvus/internal/util/nullutil" pkgcommon "github.com/milvus-io/milvus/pkg/v2/common" - "github.com/milvus-io/milvus/pkg/v2/util/funcutil" "github.com/milvus-io/milvus/pkg/v2/util/merr" "github.com/milvus-io/milvus/pkg/v2/util/parameterutil" + "github.com/milvus-io/milvus/pkg/v2/util/timestamptz" "github.com/milvus-io/milvus/pkg/v2/util/typeutil" ) @@ -446,7 +446,7 @@ func (r *rowParser) parseEntity(field *schemapb.FieldSchema, obj string, useElem } return wkbValue, nil case schemapb.DataType_Timestamptz: - tz, err := funcutil.ValidateAndReturnUnixMicroTz(obj, r.timezone) + tz, err := timestamptz.ValidateAndReturnUnixMicroTz(obj, r.timezone) if err != nil { return nil, err } diff --git a/internal/util/importutilv2/json/row_parser.go b/internal/util/importutilv2/json/row_parser.go index b5585372b2..05f172cdb4 100644 --- a/internal/util/importutilv2/json/row_parser.go +++ b/internal/util/importutilv2/json/row_parser.go @@ -27,9 +27,9 @@ import ( "github.com/milvus-io/milvus/internal/util/importutilv2/common" "github.com/milvus-io/milvus/internal/util/nullutil" pkgcommon "github.com/milvus-io/milvus/pkg/v2/common" - "github.com/milvus-io/milvus/pkg/v2/util/funcutil" "github.com/milvus-io/milvus/pkg/v2/util/merr" "github.com/milvus-io/milvus/pkg/v2/util/parameterutil" + "github.com/milvus-io/milvus/pkg/v2/util/timestamptz" "github.com/milvus-io/milvus/pkg/v2/util/typeutil" ) @@ -571,7 +571,7 @@ func (r *rowParser) parseEntity(fieldID int64, obj any) (any, error) { if !ok { return nil, r.wrapTypeError(obj, fieldID) } - tz, err := funcutil.ValidateAndReturnUnixMicroTz(strValue, r.timezone) + tz, err := timestamptz.ValidateAndReturnUnixMicroTz(strValue, r.timezone) if err != nil { return nil, err } diff --git a/internal/util/importutilv2/numpy/field_reader.go b/internal/util/importutilv2/numpy/field_reader.go index 61285931c1..234d69ff01 100644 --- a/internal/util/importutilv2/numpy/field_reader.go +++ b/internal/util/importutilv2/numpy/field_reader.go @@ -31,9 +31,9 @@ import ( "github.com/milvus-io/milvus/internal/json" "github.com/milvus-io/milvus/internal/util/importutilv2/common" pkgcommon "github.com/milvus-io/milvus/pkg/v2/common" - "github.com/milvus-io/milvus/pkg/v2/util/funcutil" "github.com/milvus-io/milvus/pkg/v2/util/merr" "github.com/milvus-io/milvus/pkg/v2/util/parameterutil" + "github.com/milvus-io/milvus/pkg/v2/util/timestamptz" "github.com/milvus-io/milvus/pkg/v2/util/typeutil" ) @@ -192,7 +192,7 @@ func (c *FieldReader) Next(count int64) (any, any, error) { } int64Ts := make([]int64, 0, len(strs)) for _, strValue := range strs { - tz, err := funcutil.ValidateAndReturnUnixMicroTz(strValue, c.timezone) + tz, err := timestamptz.ValidateAndReturnUnixMicroTz(strValue, c.timezone) if err != nil { return nil, nil, err } diff --git a/internal/util/importutilv2/parquet/field_reader.go b/internal/util/importutilv2/parquet/field_reader.go index 0228596aa6..c5de8d478d 100644 --- a/internal/util/importutilv2/parquet/field_reader.go +++ b/internal/util/importutilv2/parquet/field_reader.go @@ -33,9 +33,9 @@ import ( "github.com/milvus-io/milvus/internal/util/importutilv2/common" "github.com/milvus-io/milvus/internal/util/nullutil" pkgcommon "github.com/milvus-io/milvus/pkg/v2/common" - "github.com/milvus-io/milvus/pkg/v2/util/funcutil" "github.com/milvus-io/milvus/pkg/v2/util/merr" "github.com/milvus-io/milvus/pkg/v2/util/parameterutil" + "github.com/milvus-io/milvus/pkg/v2/util/timestamptz" "github.com/milvus-io/milvus/pkg/v2/util/typeutil" ) @@ -729,7 +729,7 @@ func ReadNullableTimestamptzData(pcr *FieldReader, count int64) (any, []bool, er // Convert the ISO 8601 string to int64 (UTC microseconds). // The pcr.timezone is used as the default timezone if the string (strValue) // does not contain an explicit UTC offset (e.g., "+08:00"). - tz, err := funcutil.ValidateAndReturnUnixMicroTz(strValue, pcr.timezone) + tz, err := timestamptz.ValidateAndReturnUnixMicroTz(strValue, pcr.timezone) if err != nil { return nil, nil, err } @@ -755,7 +755,7 @@ func ReadTimestamptzData(pcr *FieldReader, count int64) (any, error) { for _, strValue := range data.([]string) { // Convert the ISO 8601 string to int64 (UTC microseconds). // The pcr.timezone is used as the default if the string lacks an explicit offset. - tz, err := funcutil.ValidateAndReturnUnixMicroTz(strValue, pcr.timezone) + tz, err := timestamptz.ValidateAndReturnUnixMicroTz(strValue, pcr.timezone) if err != nil { return nil, err } diff --git a/pkg/go.sum b/pkg/go.sum index b73c3020e6..f38c2d1d64 100644 --- a/pkg/go.sum +++ b/pkg/go.sum @@ -349,6 +349,7 @@ github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXP github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/gogo/status v1.1.0/go.mod h1:BFv9nrluPLmrS0EmGVvLaPNmRosr9KapBYd5/hpY1WM= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= github.com/golang-jwt/jwt/v5 v5.3.0 h1:pv4AsKCKKZuqlgs5sUmn4x8UlGa0kEVt/puTpKx9vvo= github.com/golang-jwt/jwt/v5 v5.3.0/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= diff --git a/pkg/util/funcutil/timestamptz.go b/pkg/util/funcutil/timestamptz.go deleted file mode 100644 index 167a56166c..0000000000 --- a/pkg/util/funcutil/timestamptz.go +++ /dev/null @@ -1,200 +0,0 @@ -package funcutil - -import ( - "bytes" - "fmt" - "strings" - "time" -) - -// Define max/min offset boundaries in seconds for validation, exported for external checks if necessary. -const ( - MaxOffsetSeconds = 14 * 3600 // +14:00 - MinOffsetSeconds = -12 * 3600 // -12:00 -) - -// NaiveTzLayouts is a list of common timestamp formats that lack timezone information. -var NaiveTzLayouts = []string{ - "2006-01-02T15:04:05.999999999", - "2006-01-02T15:04:05", - "2006-01-02 15:04:05.999999999", - "2006-01-02 15:04:05", -} - -// ParseTimeTz is the internal core function for parsing TZ-aware or naive timestamps. -// It includes strict validation for the UTC offset range. -func ParseTimeTz(inputStr string, defaultTimezoneStr string) (time.Time, error) { - // 1. Primary parsing: Attempt to parse a TZ-aware string (RFC3339Nano) - t, err := time.Parse(time.RFC3339Nano, inputStr) - - if err == nil { - // Parsing succeeded (TZ-aware string). Now, perform the strict offset validation. - - // If the string contains an explicit offset (like +99:00), t.Zone() will reflect it. - _, offsetSeconds := t.Zone() - - if offsetSeconds > MaxOffsetSeconds || offsetSeconds < MinOffsetSeconds { - offsetHours := offsetSeconds / 3600 - return time.Time{}, fmt.Errorf("UTC offset hour %d is out of the valid range [%d, %d]", offsetHours, MinOffsetSeconds/3600, MaxOffsetSeconds/3600) - } - - return t, nil - } - - loc, err := time.LoadLocation(defaultTimezoneStr) - if err != nil { - return time.Time{}, fmt.Errorf("invalid default timezone string '%s': %w", defaultTimezoneStr, err) - } - - // 2. Fallback parsing: Attempt to parse a naive string using NaiveTzLayouts - var parsed bool - for _, layout := range NaiveTzLayouts { - // For naive strings, time.ParseInLocation assigns the default location (loc). - parsedTime, parseErr := time.ParseInLocation(layout, inputStr, loc) - if parseErr == nil { - t = parsedTime - parsed = true - break - } - } - - if !parsed { - return time.Time{}, fmt.Errorf("invalid timestamp string: '%s'. Does not match any known format", inputStr) - } - - // No offset validation needed here: The time was assigned the safe defaultTimezoneStr (loc), - // which is already validated via time.LoadLocation. - - return t, nil -} - -// ValidateTimestampTz checks if the timestamp string is valid (TZ-aware or naive + default TZ). -func ValidateTimestampTz(inputStr string, defaultTimezoneStr string) error { - _, err := ParseTimeTz(inputStr, defaultTimezoneStr) - return err -} - -// ValidateAndNormalizeTimestampTz validates the timestamp and normalizes it to a TZ-aware RFC3339Nano string. -func ValidateAndNormalizeTimestampTz(inputStr string, defaultTimezoneStr string) (string, error) { - t, err := ParseTimeTz(inputStr, defaultTimezoneStr) - if err != nil { - return "", err - } - // Normalization: Format the time object to include the timezone offset. - return t.Format(time.RFC3339Nano), nil -} - -// ValidateAndReturnUnixMicroTz validates the timestamp and returns its Unix microsecond (int64) representation. -func ValidateAndReturnUnixMicroTz(inputStr string, defaultTimezoneStr string) (int64, error) { - t, err := ParseTimeTz(inputStr, defaultTimezoneStr) - if err != nil { - return 0, err - } - // UnixMicro() returns the number of microseconds since UTC 1970-01-01T00:00:00Z. - return t.UnixMicro(), nil -} - -// CompareUnixMicroTz compares two timestamp strings at Unix microsecond precision. -// If both strings are valid and represent the same microsecond moment in time, it returns true. -// Note: It assumes the input strings are guaranteed to be valid as per the requirement. -// If not, it will return an error indicating the invalid input. -func CompareUnixMicroTz(ts1 string, ts2 string, defaultTimezoneStr string) (bool, error) { - // 1. Parse the first timestamp - t1, err := ParseTimeTz(ts1, defaultTimezoneStr) - if err != nil { - return false, fmt.Errorf("error parsing first timestamp '%s': %w", ts1, err) - } - - // 2. Parse the second timestamp - t2, err := ParseTimeTz(ts2, defaultTimezoneStr) - if err != nil { - return false, fmt.Errorf("error parsing second timestamp '%s': %w", ts2, err) - } - - // 3. Compare their Unix Microsecond values (int64) - // This automatically compares them based on the UTC epoch, regardless of their original location representation. - return t1.UnixMicro() == t2.UnixMicro(), nil -} - -// ConvertUnixMicroToTimezoneString converts a Unix microsecond timestamp (UTC epoch) -// into a TZ-aware string formatted as RFC3339Nano, adjusted to the target timezone. -func ConvertUnixMicroToTimezoneString(ts int64, targetTimezoneStr string) (string, error) { - loc, err := time.LoadLocation(targetTimezoneStr) - if err != nil { - return "", fmt.Errorf("invalid target timezone string '%s': %w", targetTimezoneStr, err) - } - - // 1. Convert Unix Microsecond (UTC) to a time.Time object (still in UTC). - t := time.UnixMicro(ts).UTC() - - // 2. Adjust the time object to the target location. - localTime := t.In(loc) - - // 3. Format the result. - return localTime.Format(time.RFC3339Nano), nil -} - -// formatTimeMicroWithoutTrailingZeros is an optimized function to format a time.Time -// object. It first truncates the time to microsecond precision (6 digits) and then -// removes all trailing zeros from the fractional seconds part. -// -// Example 1: 2025-03-20T10:30:00.123456000Z -> 2025-03-20T10:30:00.123456Z -// Example 2: 2025-03-20T10:30:00.123000000Z -> 2025-03-20T10:30:00.123Z -// Example 3: 2025-03-20T10:30:00.000000000Z -> 2025-03-20T10:30:00Z -func FormatTimeMicroWithoutTrailingZeros(t time.Time) string { - // 1. Truncate to Microsecond (6 digits max) to ensure we don't exceed the required precision. - tMicro := t.Truncate(time.Microsecond) - - // 2. Format the time using the standard high precision format (RFC3339Nano). - // This results in exactly 9 fractional digits, padded with trailing zeros if necessary. - s := tMicro.Format(time.RFC3339Nano) - - // 3. Locate the key delimiters ('.' and the Timezone marker 'Z' or '+/-'). - dotIndex := strings.LastIndexByte(s, '.') - - // Find the Timezone marker index (Z, +, or -) - tzIndex := len(s) - 1 - for ; tzIndex >= 0; tzIndex-- { - if s[tzIndex] == 'Z' || s[tzIndex] == '+' || s[tzIndex] == '-' { - break - } - } - - // If the format is unexpected, return the original string. - if dotIndex == -1 || tzIndex == -1 { - return s - } - - // 4. Extract and efficiently trim the fractional part using bytes.TrimRight. - - // Slice the fractional part (e.g., "123456000") - fractionalPart := s[dotIndex+1 : tzIndex] - - // Use bytes.TrimRight for efficient removal of trailing '0' characters. - trimmedBytes := bytes.TrimRight([]byte(fractionalPart), "0") - - // 5. Reconstruct the final string based on the trimming result. - - // Case A: The fractional part was entirely zeros (e.g., .000000000) - if len(trimmedBytes) == 0 { - // Remove the '.' and the fractional part, keep the Timezone marker. - // Result: "2025-03-20T10:30:00Z" - return s[:dotIndex] + s[tzIndex:] - } - - // Case B: Fractional part remains (e.g., .123, .123456) - // Recombine: [Time Body] + "." + [Trimmed Fraction] + [Timezone Marker] - // The dot (s[:dotIndex+1]) must be retained here. - return s[:dotIndex+1] + string(trimmedBytes) + s[tzIndex:] -} - -// IsTimezoneValid checks if a given string is a valid, recognized timezone name -// (e.g., "Asia/Shanghai" or "UTC"). -// It utilizes Go's time.LoadLocation function. -func IsTimezoneValid(tz string) bool { - if tz == "" { - return false - } - _, err := time.LoadLocation(tz) - return err == nil -} diff --git a/pkg/util/timestamptz/timestamptz.go b/pkg/util/timestamptz/timestamptz.go new file mode 100644 index 0000000000..7bfd7bb682 --- /dev/null +++ b/pkg/util/timestamptz/timestamptz.go @@ -0,0 +1,385 @@ +package timestamptz + +import ( + "bytes" + "fmt" + "strings" + "time" + + "github.com/cockroachdb/errors" + + "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" + "github.com/milvus-io/milvus/pkg/v2/common" + "github.com/milvus-io/milvus/pkg/v2/util/funcutil" +) + +// Define max/min offset boundaries in seconds for validation, exported for external checks if necessary. +const ( + MaxOffsetSeconds = 14 * 3600 // +14:00 + MinOffsetSeconds = -12 * 3600 // -12:00 +) + +// NaiveTzLayouts is a list of common timestamp formats that lack timezone information. +var NaiveTzLayouts = []string{ + "2006-01-02T15:04:05.999999999", + "2006-01-02T15:04:05", + "2006-01-02 15:04:05.999999999", + "2006-01-02 15:04:05", +} + +// ParseTimeTz is the internal core function for parsing TZ-aware or naive timestamps. +// It includes strict validation for the UTC offset range. +func ParseTimeTz(inputStr string, defaultTimezoneStr string) (time.Time, error) { + // 1. Primary parsing: Attempt to parse a TZ-aware string (RFC3339Nano) + t, err := time.Parse(time.RFC3339Nano, inputStr) + + if err == nil { + // Parsing succeeded (TZ-aware string). Now, perform the strict offset validation. + + // If the string contains an explicit offset (like +99:00), t.Zone() will reflect it. + _, offsetSeconds := t.Zone() + + if offsetSeconds > MaxOffsetSeconds || offsetSeconds < MinOffsetSeconds { + offsetHours := offsetSeconds / 3600 + return time.Time{}, fmt.Errorf("UTC offset hour %d is out of the valid range [%d, %d]", offsetHours, MinOffsetSeconds/3600, MaxOffsetSeconds/3600) + } + + return t, nil + } + + loc, err := time.LoadLocation(defaultTimezoneStr) + if err != nil { + return time.Time{}, fmt.Errorf("invalid default timezone string '%s': %w", defaultTimezoneStr, err) + } + + // 2. Fallback parsing: Attempt to parse a naive string using NaiveTzLayouts + var parsed bool + for _, layout := range NaiveTzLayouts { + // For naive strings, time.ParseInLocation assigns the default location (loc). + parsedTime, parseErr := time.ParseInLocation(layout, inputStr, loc) + if parseErr == nil { + t = parsedTime + parsed = true + break + } + } + + if !parsed { + return time.Time{}, fmt.Errorf("invalid timestamp string: '%s'. Does not match any known format", inputStr) + } + + // No offset validation needed here: The time was assigned the safe defaultTimezoneStr (loc), + // which is already validated via time.LoadLocation. + + return t, nil +} + +// ValidateTimestampTz checks if the timestamp string is valid (TZ-aware or naive + default TZ). +func ValidateTimestampTz(inputStr string, defaultTimezoneStr string) error { + _, err := ParseTimeTz(inputStr, defaultTimezoneStr) + return err +} + +// ValidateAndNormalizeTimestampTz validates the timestamp and normalizes it to a TZ-aware RFC3339Nano string. +func ValidateAndNormalizeTimestampTz(inputStr string, defaultTimezoneStr string) (string, error) { + t, err := ParseTimeTz(inputStr, defaultTimezoneStr) + if err != nil { + return "", err + } + // Normalization: Format the time object to include the timezone offset. + return t.Format(time.RFC3339Nano), nil +} + +// ValidateAndReturnUnixMicroTz validates the timestamp and returns its Unix microsecond (int64) representation. +func ValidateAndReturnUnixMicroTz(inputStr string, defaultTimezoneStr string) (int64, error) { + t, err := ParseTimeTz(inputStr, defaultTimezoneStr) + if err != nil { + return 0, err + } + // UnixMicro() returns the number of microseconds since UTC 1970-01-01T00:00:00Z. + return t.UnixMicro(), nil +} + +// CompareUnixMicroTz compares two timestamp strings at Unix microsecond precision. +// If both strings are valid and represent the same microsecond moment in time, it returns true. +// Note: It assumes the input strings are guaranteed to be valid as per the requirement. +// If not, it will return an error indicating the invalid input. +func CompareUnixMicroTz(ts1 string, ts2 string, defaultTimezoneStr string) (bool, error) { + // 1. Parse the first timestamp + t1, err := ParseTimeTz(ts1, defaultTimezoneStr) + if err != nil { + return false, fmt.Errorf("error parsing first timestamp '%s': %w", ts1, err) + } + + // 2. Parse the second timestamp + t2, err := ParseTimeTz(ts2, defaultTimezoneStr) + if err != nil { + return false, fmt.Errorf("error parsing second timestamp '%s': %w", ts2, err) + } + + // 3. Compare their Unix Microsecond values (int64) + // This automatically compares them based on the UTC epoch, regardless of their original location representation. + return t1.UnixMicro() == t2.UnixMicro(), nil +} + +// ConvertUnixMicroToTimezoneString converts a Unix microsecond timestamp (UTC epoch) +// into a TZ-aware string formatted as RFC3339Nano, adjusted to the target timezone. +func ConvertUnixMicroToTimezoneString(ts int64, targetTimezoneStr string) (string, error) { + loc, err := time.LoadLocation(targetTimezoneStr) + if err != nil { + return "", fmt.Errorf("invalid target timezone string '%s': %w", targetTimezoneStr, err) + } + + // 1. Convert Unix Microsecond (UTC) to a time.Time object (still in UTC). + t := time.UnixMicro(ts).UTC() + + // 2. Adjust the time object to the target location. + localTime := t.In(loc) + + // 3. Format the result. + return localTime.Format(time.RFC3339Nano), nil +} + +// formatTimeMicroWithoutTrailingZeros is an optimized function to format a time.Time +// object. It first truncates the time to microsecond precision (6 digits) and then +// removes all trailing zeros from the fractional seconds part. +// +// Example 1: 2025-03-20T10:30:00.123456000Z -> 2025-03-20T10:30:00.123456Z +// Example 2: 2025-03-20T10:30:00.123000000Z -> 2025-03-20T10:30:00.123Z +// Example 3: 2025-03-20T10:30:00.000000000Z -> 2025-03-20T10:30:00Z +func FormatTimeMicroWithoutTrailingZeros(t time.Time) string { + // 1. Truncate to Microsecond (6 digits max) to ensure we don't exceed the required precision. + tMicro := t.Truncate(time.Microsecond) + + // 2. Format the time using the standard high precision format (RFC3339Nano). + // This results in exactly 9 fractional digits, padded with trailing zeros if necessary. + s := tMicro.Format(time.RFC3339Nano) + + // 3. Locate the key delimiters ('.' and the Timezone marker 'Z' or '+/-'). + dotIndex := strings.LastIndexByte(s, '.') + + // Find the Timezone marker index (Z, +, or -) + tzIndex := len(s) - 1 + for ; tzIndex >= 0; tzIndex-- { + if s[tzIndex] == 'Z' || s[tzIndex] == '+' || s[tzIndex] == '-' { + break + } + } + + // If the format is unexpected, return the original string. + if dotIndex == -1 || tzIndex == -1 { + return s + } + + // 4. Extract and efficiently trim the fractional part using bytes.TrimRight. + + // Slice the fractional part (e.g., "123456000") + fractionalPart := s[dotIndex+1 : tzIndex] + + // Use bytes.TrimRight for efficient removal of trailing '0' characters. + trimmedBytes := bytes.TrimRight([]byte(fractionalPart), "0") + + // 5. Reconstruct the final string based on the trimming result. + + // Case A: The fractional part was entirely zeros (e.g., .000000000) + if len(trimmedBytes) == 0 { + // Remove the '.' and the fractional part, keep the Timezone marker. + // Result: "2025-03-20T10:30:00Z" + return s[:dotIndex] + s[tzIndex:] + } + + // Case B: Fractional part remains (e.g., .123, .123456) + // Recombine: [Time Body] + "." + [Trimmed Fraction] + [Timezone Marker] + // The dot (s[:dotIndex+1]) must be retained here. + return s[:dotIndex+1] + string(trimmedBytes) + s[tzIndex:] +} + +// IsTimezoneValid checks if a given string is a valid, recognized timezone name +// (e.g., "Asia/Shanghai" or "UTC"). +// It utilizes Go's time.LoadLocation function. +func IsTimezoneValid(tz string) bool { + if tz == "" { + return false + } + _, err := time.LoadLocation(tz) + return err == nil +} + +// CheckAndRewriteTimestampTzDefaultValue processes the collection schema to validate +// and rewrite default values for TIMESTAMPTZ fields. +// +// Background: +// 1. TIMESTAMPTZ default values are initially stored as user-provided ISO 8601 strings +// (in ValueField.GetStringData()). +// 2. Milvus stores TIMESTAMPTZ data internally as UTC microseconds (int64). +// +// Logic: +// The function iterates through all fields of type DataType_Timestamptz. For each field +// with a default value: +// 1. It retrieves the collection's default timezone if no offset is present in the string. +// 2. It calls ValidateAndReturnUnixMicroTz to validate the string (including the UTC +// offset range check) and convert it to the absolute UTC microsecond (int64) value. +// 3. It rewrites the ValueField, setting the LongData field with the calculated int64 +// value, thereby replacing the initial string representation. +func CheckAndRewriteTimestampTzDefaultValue(schema *schemapb.CollectionSchema) error { + // 1. Get the collection-level default timezone. + // Assuming common.TimezoneKey and common.DefaultTimezone are defined constants. + timezone, exist := funcutil.TryGetAttrByKeyFromRepeatedKV(common.TimezoneKey, schema.GetProperties()) + if !exist { + timezone = common.DefaultTimezone + } + + for _, fieldSchema := range schema.GetFields() { + // Only process TIMESTAMPTZ fields. + if fieldSchema.GetDataType() != schemapb.DataType_Timestamptz { + continue + } + + defaultValue := fieldSchema.GetDefaultValue() + if defaultValue == nil { + continue + } + + // 2. Read the default value as a string (the input format). + // We expect the default value to be set in string_data initially. + stringTz := defaultValue.GetStringData() + if stringTz == "" { + // Skip or handle empty string default values if necessary. + continue + } + + // 3. Validate the string and convert it to UTC microsecond (int64). + // This also performs the critical UTC offset range validation. + utcMicro, err := ValidateAndReturnUnixMicroTz(stringTz, timezone) + if err != nil { + // If validation fails (e.g., invalid format or illegal offset), return error immediately. + return err + } + + // 4. Rewrite the default value to store the UTC microsecond (int64). + // By setting ValueField_LongData, the oneof field in the protobuf structure + // automatically switches from string_data to timestamptz_data(int64). + defaultValue.Data = &schemapb.ValueField_TimestamptzData{ + TimestamptzData: utcMicro, + } + + // The original string_data field is now cleared due to the oneof nature, + // and the default value is correctly represented as an int64 microsecond value. + } + return nil +} + +// CheckAndRewriteTimestampTzDefaultValueForFieldSchema processes a single FieldSchema +// to validate and rewrite the default value specifically for TIMESTAMPTZ fields. +// +// The function ensures the default value (initially a string) is correctly converted +// and stored internally as an absolute UTC microsecond (int64) value. +// +// Parameters: +// +// fieldSchema: The specific FieldSchema object to be processed. +// collectionTimezone: The collection-level default timezone string (e.g., "UTC", "Asia/Shanghai") +// used to parse timestamps without an explicit offset. +// +// Returns: +// +// error: An error if validation fails (e.g., invalid timestamp format or illegal offset range), otherwise nil. +func CheckAndRewriteTimestampTzDefaultValueForFieldSchema( + fieldSchema *schemapb.FieldSchema, + collectionTimezone string, +) error { + defaultValue := fieldSchema.GetDefaultValue() + if defaultValue == nil { + return nil + } + // log.Info("czsKKK111") + + // 2. Read the default value as a string (the initial user-provided format). + // The default value is expected to be stored in string_data initially. + stringTz := defaultValue.GetStringData() + if stringTz == "" { + // Skip or handle empty string default values if necessary. + // log.Info("czsKKK222") + return nil + } + + // 3. Validate the string and convert it to UTC microsecond (int64). + // The validation function also applies the collectionTimezone if no offset is present + // in the input stringTz, and performs offset range checks. + utcMicro, err := ValidateAndReturnUnixMicroTz(stringTz, collectionTimezone) + if err != nil { + // log.Info("czsKKK333") + + // If validation fails (e.g., invalid format or illegal offset), return error immediately. + return err + } + + // 4. Rewrite the default value to store the absolute UTC microsecond (int64). + // By setting ValueField_LongData, the oneof field in the protobuf structure + // automatically switches the internal representation from string_data to timestamptz_data(int64). + defaultValue.Data = &schemapb.ValueField_TimestamptzData{ + TimestamptzData: utcMicro, + } + fieldSchema.DefaultValue = defaultValue + // log.Info("czsKKK444", zap.Any("utc", fieldSchema.GetDefaultValue())) + return nil +} + +// RewriteTimestampTzDefaultValueToString converts the default_value of TIMESTAMPTZ fields +// in the DescribeCollectionResponse from the internal int64 (UTC microsecond) format +// back to a human-readable, timezone-aware string (RFC3339Nano). +// +// This is necessary because TIMESTAMPTZ default values are stored internally as int64 +// after validation but must be returned to the user as a string, respecting the +// collection's default timezone for display purposes if no explicit offset was stored. +func RewriteTimestampTzDefaultValueToString(schema *schemapb.CollectionSchema) error { + if schema == nil { + return nil + } + + // 1. Determine the target timezone for display. + // This is typically stored in the collection properties. + timezone, exist := funcutil.TryGetAttrByKeyFromRepeatedKV(common.TimezoneKey, schema.GetProperties()) + if !exist { + timezone = common.DefaultTimezone // Fallback to a default, like "UTC" + } + + // 2. Iterate through all fields in the schema. + for _, fieldSchema := range schema.GetFields() { + // Only process TIMESTAMPTZ fields. + if fieldSchema.GetDataType() != schemapb.DataType_Timestamptz { + continue + } + + defaultValue := fieldSchema.GetDefaultValue() + if defaultValue == nil { + continue + } + + // 3. Check if the default value is stored in the internal int64 (LongData) format. + // If it's not LongData, we assume it's either unset or already a string (which shouldn't happen + // if the creation flow worked correctly). + utcMicro, ok := defaultValue.GetData().(*schemapb.ValueField_TimestamptzData) + if !ok { + continue // Skip if not stored as LongData (int64) + } + + ts := utcMicro.TimestamptzData + + // 4. Convert the int64 microsecond value back to a timezone-aware string. + tzString, err := ConvertUnixMicroToTimezoneString(ts, timezone) + if err != nil { + // In a real system, you might log the error and use the raw int64 as a fallback string, + // but here we'll set a placeholder string to avoid crashing. + tzString = fmt.Sprintf("error converting timestamp: %v", err) + return errors.Wrap(err, tzString) + } + + // 5. Rewrite the default value field in the response schema. + // The protobuf oneof structure ensures setting one field clears the others. + fieldSchema.GetDefaultValue().Data = &schemapb.ValueField_StringData{ + StringData: tzString, + } + } + return nil +} diff --git a/pkg/util/funcutil/timestamptz_test.go b/pkg/util/timestamptz/timestamptz_test.go similarity index 99% rename from pkg/util/funcutil/timestamptz_test.go rename to pkg/util/timestamptz/timestamptz_test.go index 90cff6577b..1ff1c80e46 100644 --- a/pkg/util/funcutil/timestamptz_test.go +++ b/pkg/util/timestamptz/timestamptz_test.go @@ -1,4 +1,4 @@ -package funcutil +package timestamptz import ( "testing"