From c603f1c244caeb180945c11eef6e4e997f53e3ea Mon Sep 17 00:00:00 2001 From: jaime Date: Tue, 29 Aug 2023 14:36:26 +0800 Subject: [PATCH] Remove mysql metastore (#26633) Signed-off-by: jaime --- ci/jenkins/Nightly.groovy | 7 - cmd/tools/config/generate.go | 4 - cmd/tools/migration/configs/config.go | 9 - cmd/tools/migration/meta/210_to_220.go | 2 +- configs/milvus.yaml | 12 +- go.mod | 2 - go.sum | 4 - internal/metastore/db/dao/collection.go | 119 - internal/metastore/db/dao/collection_alias.go | 79 - .../metastore/db/dao/collection_alias_test.go | 205 -- .../metastore/db/dao/collection_channel.go | 36 - .../db/dao/collection_channel_test.go | 106 - internal/metastore/db/dao/collection_test.go | 495 ---- internal/metastore/db/dao/common.go | 62 - internal/metastore/db/dao/field.go | 36 - internal/metastore/db/dao/field_test.go | 119 - internal/metastore/db/dao/gran_id.go | 55 - internal/metastore/db/dao/gran_id_test.go | 236 -- internal/metastore/db/dao/grant.go | 49 - internal/metastore/db/dao/grant_test.go | 203 -- internal/metastore/db/dao/index.go | 93 - internal/metastore/db/dao/index_test.go | 248 -- internal/metastore/db/dao/partition.go | 57 - internal/metastore/db/dao/partition_test.go | 155 -- internal/metastore/db/dao/role.go | 48 - internal/metastore/db/dao/role_test.go | 133 -- internal/metastore/db/dao/segment_index.go | 125 - .../metastore/db/dao/segment_index_test.go | 248 -- internal/metastore/db/dao/user.go | 75 - internal/metastore/db/dao/user_role.go | 49 - internal/metastore/db/dao/user_role_test.go | 201 -- internal/metastore/db/dao/user_test.go | 212 -- internal/metastore/db/dbcore/core.go | 119 - internal/metastore/db/dbmodel/base_model.go | 11 - internal/metastore/db/dbmodel/collection.go | 107 - .../metastore/db/dbmodel/collection_alias.go | 30 - .../db/dbmodel/collection_channel.go | 40 - .../metastore/db/dbmodel/collection_test.go | 106 - internal/metastore/db/dbmodel/common.go | 30 - internal/metastore/db/dbmodel/field.go | 91 - internal/metastore/db/dbmodel/grant.go | 20 - internal/metastore/db/dbmodel/grant_id.go | 21 - internal/metastore/db/dbmodel/index.go | 127 -- .../db/dbmodel/mocks/ICollAliasDb.go | 109 - .../db/dbmodel/mocks/ICollChannelDb.go | 65 - .../db/dbmodel/mocks/ICollectionDb.go | 146 -- .../metastore/db/dbmodel/mocks/IFieldDb.go | 65 - .../metastore/db/dbmodel/mocks/IGrantDb.go | 79 - .../metastore/db/dbmodel/mocks/IGrantIDDb.go | 79 - .../metastore/db/dbmodel/mocks/IIndexDb.go | 130 -- .../metastore/db/dbmodel/mocks/IMetaDomain.go | 222 -- .../db/dbmodel/mocks/IPartitionDb.go | 79 - .../metastore/db/dbmodel/mocks/IRoleDb.go | 79 - .../db/dbmodel/mocks/ISegmentIndexDb.go | 144 -- .../metastore/db/dbmodel/mocks/IUserDb.go | 116 - .../metastore/db/dbmodel/mocks/IUserRoleDb.go | 79 - internal/metastore/db/dbmodel/partition.go | 54 - internal/metastore/db/dbmodel/role.go | 23 - .../metastore/db/dbmodel/segment_index.go | 99 - internal/metastore/db/dbmodel/user.go | 44 - internal/metastore/db/dbmodel/user_role.go | 21 - .../metastore/db/rootcoord/table_catalog.go | 1003 -------- .../db/rootcoord/table_catalog_test.go | 2015 ----------------- internal/rootcoord/root_coord.go | 11 - pkg/go.sum | 1 - pkg/util/constant.go | 1 - pkg/util/paramtable/base_table.go | 2 +- pkg/util/paramtable/service_param.go | 75 +- .../milvus_benchmark/env/helm_utils.py | 24 +- .../suites/011_cluster_cpu_accuracy_ann.yaml | 22 +- .../suites/011_cpu_accuracy_ann_debug.yaml | 3 +- .../suites/011_cpu_search_sift10m.yaml | 2 +- .../suites/011_gpu_search_debug.yaml | 2 +- .../suites/011_insert_debug.yaml | 2 +- .../suites/cluster_locust_mix.yaml | 2 +- .../suites/debug_gpu_search.yaml | 2 +- .../suites/locust_search.yaml | 2 +- .../testcases/stability/test_mysql.py | 45 - tests/python_client/testcases/test_alias.py | 1 - tests/python_client/utils/util_pymilvus.py | 4 +- tests/scripts/values/mysql.yaml | 274 --- 81 files changed, 27 insertions(+), 9285 deletions(-) delete mode 100644 internal/metastore/db/dao/collection.go delete mode 100644 internal/metastore/db/dao/collection_alias.go delete mode 100644 internal/metastore/db/dao/collection_alias_test.go delete mode 100644 internal/metastore/db/dao/collection_channel.go delete mode 100644 internal/metastore/db/dao/collection_channel_test.go delete mode 100644 internal/metastore/db/dao/collection_test.go delete mode 100644 internal/metastore/db/dao/common.go delete mode 100644 internal/metastore/db/dao/field.go delete mode 100644 internal/metastore/db/dao/field_test.go delete mode 100644 internal/metastore/db/dao/gran_id.go delete mode 100644 internal/metastore/db/dao/gran_id_test.go delete mode 100644 internal/metastore/db/dao/grant.go delete mode 100644 internal/metastore/db/dao/grant_test.go delete mode 100644 internal/metastore/db/dao/index.go delete mode 100644 internal/metastore/db/dao/index_test.go delete mode 100644 internal/metastore/db/dao/partition.go delete mode 100644 internal/metastore/db/dao/partition_test.go delete mode 100644 internal/metastore/db/dao/role.go delete mode 100644 internal/metastore/db/dao/role_test.go delete mode 100644 internal/metastore/db/dao/segment_index.go delete mode 100644 internal/metastore/db/dao/segment_index_test.go delete mode 100644 internal/metastore/db/dao/user.go delete mode 100644 internal/metastore/db/dao/user_role.go delete mode 100644 internal/metastore/db/dao/user_role_test.go delete mode 100644 internal/metastore/db/dao/user_test.go delete mode 100644 internal/metastore/db/dbcore/core.go delete mode 100644 internal/metastore/db/dbmodel/base_model.go delete mode 100644 internal/metastore/db/dbmodel/collection.go delete mode 100644 internal/metastore/db/dbmodel/collection_alias.go delete mode 100644 internal/metastore/db/dbmodel/collection_channel.go delete mode 100644 internal/metastore/db/dbmodel/collection_test.go delete mode 100644 internal/metastore/db/dbmodel/common.go delete mode 100644 internal/metastore/db/dbmodel/field.go delete mode 100644 internal/metastore/db/dbmodel/grant.go delete mode 100644 internal/metastore/db/dbmodel/grant_id.go delete mode 100644 internal/metastore/db/dbmodel/index.go delete mode 100644 internal/metastore/db/dbmodel/mocks/ICollAliasDb.go delete mode 100644 internal/metastore/db/dbmodel/mocks/ICollChannelDb.go delete mode 100644 internal/metastore/db/dbmodel/mocks/ICollectionDb.go delete mode 100644 internal/metastore/db/dbmodel/mocks/IFieldDb.go delete mode 100644 internal/metastore/db/dbmodel/mocks/IGrantDb.go delete mode 100644 internal/metastore/db/dbmodel/mocks/IGrantIDDb.go delete mode 100644 internal/metastore/db/dbmodel/mocks/IIndexDb.go delete mode 100644 internal/metastore/db/dbmodel/mocks/IMetaDomain.go delete mode 100644 internal/metastore/db/dbmodel/mocks/IPartitionDb.go delete mode 100644 internal/metastore/db/dbmodel/mocks/IRoleDb.go delete mode 100644 internal/metastore/db/dbmodel/mocks/ISegmentIndexDb.go delete mode 100644 internal/metastore/db/dbmodel/mocks/IUserDb.go delete mode 100644 internal/metastore/db/dbmodel/mocks/IUserRoleDb.go delete mode 100644 internal/metastore/db/dbmodel/partition.go delete mode 100644 internal/metastore/db/dbmodel/role.go delete mode 100644 internal/metastore/db/dbmodel/segment_index.go delete mode 100644 internal/metastore/db/dbmodel/user.go delete mode 100644 internal/metastore/db/dbmodel/user_role.go delete mode 100644 internal/metastore/db/rootcoord/table_catalog.go delete mode 100644 internal/metastore/db/rootcoord/table_catalog_test.go delete mode 100644 tests/python_client/testcases/stability/test_mysql.py delete mode 100644 tests/scripts/values/mysql.yaml diff --git a/ci/jenkins/Nightly.groovy b/ci/jenkins/Nightly.groovy index 2facccb7ac..ed327b5d1c 100644 --- a/ci/jenkins/Nightly.groovy +++ b/ci/jenkins/Nightly.groovy @@ -103,18 +103,14 @@ pipeline { script { sh 'printenv' def clusterEnabled = "false" - def mysqlEnabled = "false" // def setMemoryResourceLimitArgs="--set standalone.resources.limits.memory=4Gi" def mqMode='pulsar' // default using is pulsar - def mysql_architecture = "standalone" def authenticationEnabled = "false" if ("${MILVUS_SERVER_TYPE}" == "distributed-pulsar") { clusterEnabled = "true" } else if ("${MILVUS_SERVER_TYPE}" == "distributed-kafka") { clusterEnabled = "true" -// mysqlEnabled = "true" mqMode='kafka' -// mysql_architecture = "replication" } else if("${MILVUS_SERVER_TYPE}" == "standalone-authentication") { authenticationEnabled = "true" } @@ -148,8 +144,6 @@ pipeline { --set queryNode.replicas=2 \ --set indexNode.replicas=2 \ --set dataNode.replicas=2 \ - --set mysql.enabled=${mysqlEnabled} \ - --set mysql.architecture=${mysql_architecture} \ --set dataCoordinator.gc.missingTolerance=86400 \ --set dataCoordinator.gc.dropTolerance=86400 \ --set indexCoordinator.gc.interval=1 \ @@ -167,7 +161,6 @@ pipeline { --set common.security.authorizationEnabled=${authenticationEnabled} \ --version ${chart_version} \ -f values/${mqMode}.yaml \ - -f values/mysql.yaml \ -f values/ci/nightly.yaml " """ } diff --git a/cmd/tools/config/generate.go b/cmd/tools/config/generate.go index a1a127d529..65c0f256d1 100644 --- a/cmd/tools/config/generate.go +++ b/cmd/tools/config/generate.go @@ -206,10 +206,6 @@ func WriteYaml() { { name: "metastore", }, - { - name: "mysql", - header: "\n# Related configuration of mysql, used to store Milvus metadata.", - }, { name: "localStorage", }, diff --git a/cmd/tools/migration/configs/config.go b/cmd/tools/migration/configs/config.go index 15511d8adc..8a1b59df62 100644 --- a/cmd/tools/migration/configs/config.go +++ b/cmd/tools/migration/configs/config.go @@ -66,7 +66,6 @@ func (c *RunConfig) init(base *paramtable.BaseTable) { type MilvusConfig struct { MetaStoreCfg *paramtable.MetaStoreConfig EtcdCfg *paramtable.EtcdConfig - MysqlCfg *paramtable.MetaDBConfig } func newMilvusConfig(base *paramtable.BaseTable) *MilvusConfig { @@ -78,16 +77,8 @@ func newMilvusConfig(base *paramtable.BaseTable) *MilvusConfig { func (c *MilvusConfig) init(base *paramtable.BaseTable) { c.MetaStoreCfg = ¶mtable.MetaStoreConfig{} c.EtcdCfg = ¶mtable.EtcdConfig{} - c.MysqlCfg = ¶mtable.MetaDBConfig{} c.MetaStoreCfg.Init(base) - - switch c.MetaStoreCfg.MetaStoreType.GetValue() { - case util.MetaStoreTypeMysql: - c.MysqlCfg.Init(base) - default: - } - c.EtcdCfg.Init(base) } diff --git a/cmd/tools/migration/meta/210_to_220.go b/cmd/tools/migration/meta/210_to_220.go index 0c67f0f204..3ae5ee7722 100644 --- a/cmd/tools/migration/meta/210_to_220.go +++ b/cmd/tools/migration/meta/210_to_220.go @@ -183,7 +183,7 @@ func combineToCollectionIndexesMeta220(fieldIndexes FieldIndexes210, collectionI newIndexName = "_default_idx_" + strconv.FormatInt(index.GetFiledID(), 10) } record := &model.Index{ - TenantID: "", // TODO: how to set this if we support mysql later? + TenantID: "", CollectionID: collectionID, FieldID: index.GetFiledID(), IndexID: index.GetIndexID(), diff --git a/configs/milvus.yaml b/configs/milvus.yaml index c674dd737d..033fdc2b4e 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -45,19 +45,9 @@ etcd: metastore: # Default value: etcd - # Valid values: [etcd, mysql] + # Valid values: etcd type: etcd -# Related configuration of mysql, used to store Milvus metadata. -mysql: - username: root - password: 123456 - address: localhost - port: 3306 - dbName: milvus_meta - maxOpenConns: 20 - maxIdleConns: 5 - localStorage: path: /var/lib/milvus/data/ # please adjust in embedded Milvus: /tmp/milvus/data/ diff --git a/go.mod b/go.mod index 6324e611d1..96c2bf491a 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,6 @@ require ( golang.org/x/text v0.9.0 google.golang.org/grpc v1.54.0 google.golang.org/grpc/examples v0.0.0-20220617181431-3e7b97febc7f - gorm.io/driver/mysql v1.3.5 gorm.io/gorm v1.23.8 stathat.com/c/consistent v1.0.0 ) @@ -105,7 +104,6 @@ require ( github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect github.com/go-playground/validator/v10 v10.14.0 // indirect - github.com/go-sql-driver/mysql v1.6.0 // indirect github.com/goccy/go-json v0.10.2 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/godbus/dbus/v5 v5.0.4 // indirect diff --git a/go.sum b/go.sum index 553865ab6e..3ed9362bb2 100644 --- a/go.sum +++ b/go.sum @@ -288,8 +288,6 @@ github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91 github.com/go-playground/validator/v10 v10.14.0 h1:vgvQWe3XCz3gIeFDm/HnTIbj6UGmg/+t63MyGU2n5js= github.com/go-playground/validator/v10 v10.14.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= -github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= -github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= @@ -1449,8 +1447,6 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gorm.io/driver/mysql v1.3.5 h1:iWBTVW/8Ij5AG4e0G/zqzaJblYkBI1VIL1LG2HUGsvY= -gorm.io/driver/mysql v1.3.5/go.mod h1:sSIebwZAVPiT+27jK9HIwvsqOGKx3YMPmrA3mBJR10c= gorm.io/gorm v1.23.8 h1:h8sGJ+biDgBA1AD1Ha9gFCx7h8npU7AsLdlkX0n2TpE= gorm.io/gorm v1.23.8/go.mod h1:l2lP/RyAtc1ynaTjFksBde/O8v9oOGIApu2/xRitmZk= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/metastore/db/dao/collection.go b/internal/metastore/db/dao/collection.go deleted file mode 100644 index a77c9c253a..0000000000 --- a/internal/metastore/db/dao/collection.go +++ /dev/null @@ -1,119 +0,0 @@ -package dao - -import ( - "fmt" - - "github.com/cockroachdb/errors" - "go.uber.org/zap" - "gorm.io/gorm" - "gorm.io/gorm/clause" - - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - "github.com/milvus-io/milvus/pkg/log" - "github.com/milvus-io/milvus/pkg/util/typeutil" -) - -type collectionDb struct { - db *gorm.DB -} - -func (s *collectionDb) GetCollectionIDTs(tenantID string, collectionID typeutil.UniqueID, ts typeutil.Timestamp) (*dbmodel.Collection, error) { - var col dbmodel.Collection - - err := s.db.Model(&dbmodel.Collection{}).Select("collection_id, ts").Where("tenant_id = ? AND collection_id = ? AND ts <= ?", tenantID, collectionID, ts).Order("ts desc").Take(&col).Error - - if errors.Is(err, gorm.ErrRecordNotFound) { - log.Warn("record not found", zap.Int64("collectionID", collectionID), zap.Uint64("ts", ts), zap.Error(err)) - return nil, fmt.Errorf("record not found, collID=%d, ts=%d", collectionID, ts) - } - if err != nil { - log.Error("get collection ts failed", zap.String("tenant", tenantID), zap.Int64("collectionID", collectionID), zap.Uint64("ts", ts), zap.Error(err)) - return nil, err - } - - return &col, nil -} - -func (s *collectionDb) ListCollectionIDTs(tenantID string, ts typeutil.Timestamp) ([]*dbmodel.Collection, error) { - var r []*dbmodel.Collection - - err := s.db.Model(&dbmodel.Collection{}).Select("collection_id, MAX(ts) ts").Where("tenant_id = ? AND ts <= ?", tenantID, ts).Group("collection_id").Find(&r).Error - if err != nil { - log.Error("list collection_id & latest ts pairs in collections failed", zap.String("tenant", tenantID), zap.Uint64("ts", ts), zap.Error(err)) - return nil, err - } - - return r, nil -} - -func (s *collectionDb) Get(tenantID string, collectionID typeutil.UniqueID, ts typeutil.Timestamp) (*dbmodel.Collection, error) { - var r dbmodel.Collection - - err := s.db.Model(&dbmodel.Collection{}).Where("tenant_id = ? AND collection_id = ? AND ts = ? AND is_deleted = false", tenantID, collectionID, ts).Take(&r).Error - - if errors.Is(err, gorm.ErrRecordNotFound) { - return nil, fmt.Errorf("collection not found, collID=%d, ts=%d", collectionID, ts) - } - if err != nil { - log.Error("get collection by collection_id and ts failed", zap.String("tenant", tenantID), zap.Int64("collectionID", collectionID), zap.Uint64("ts", ts), zap.Error(err)) - return nil, err - } - - return &r, nil -} - -func (s *collectionDb) GetCollectionIDByName(tenantID string, collectionName string, ts typeutil.Timestamp) (typeutil.UniqueID, error) { - var r dbmodel.Collection - - err := s.db.Model(&dbmodel.Collection{}).Select("collection_id").Where("tenant_id = ? AND collection_name = ? AND ts <= ?", tenantID, collectionName, ts).Order("ts desc").Take(&r).Error - - if errors.Is(err, gorm.ErrRecordNotFound) { - return 0, fmt.Errorf("get collection_id by collection_name not found, collName=%s, ts=%d", collectionName, ts) - } - if err != nil { - log.Error("get collection_id by collection_name failed", zap.String("tenant", tenantID), zap.String("collName", collectionName), zap.Uint64("ts", ts), zap.Error(err)) - return 0, err - } - - return r.CollectionID, nil -} - -// Insert used in create & drop collection, needs be an idempotent operation, so we use DoNothing strategy here so it will not throw exception for retry, equivalent to kv catalog -func (s *collectionDb) Insert(in *dbmodel.Collection) error { - err := s.db.Clauses(clause.OnConflict{ - // constraint UNIQUE (tenant_id, collection_id, ts) - DoNothing: true, - }).Create(&in).Error - - if err != nil { - log.Error("insert collection failed", zap.String("tenant", in.TenantID), zap.Int64("collectionID", in.CollectionID), zap.Uint64("ts", in.Ts), zap.Error(err)) - return err - } - - return nil -} - -func generateCollectionUpdatesWithoutID(in *dbmodel.Collection) map[string]interface{} { - ret := map[string]interface{}{ - "tenant_id": in.TenantID, - "collection_id": in.CollectionID, - "collection_name": in.CollectionName, - "description": in.Description, - "auto_id": in.AutoID, - "shards_num": in.ShardsNum, - "start_position": in.StartPosition, - "consistency_level": in.ConsistencyLevel, - "status": in.Status, - "properties": in.Properties, - "ts": in.Ts, - "is_deleted": in.IsDeleted, - "created_at": in.CreatedAt, - "updated_at": in.UpdatedAt, - } - return ret -} - -func (s *collectionDb) Update(in *dbmodel.Collection) error { - updates := generateCollectionUpdatesWithoutID(in) - return s.db.Model(&dbmodel.Collection{}).Where("id = ?", in.ID).Updates(updates).Error -} diff --git a/internal/metastore/db/dao/collection_alias.go b/internal/metastore/db/dao/collection_alias.go deleted file mode 100644 index 2e6e6439f8..0000000000 --- a/internal/metastore/db/dao/collection_alias.go +++ /dev/null @@ -1,79 +0,0 @@ -package dao - -import ( - "fmt" - - "github.com/cockroachdb/errors" - "go.uber.org/zap" - "gorm.io/gorm" - "gorm.io/gorm/clause" - - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - "github.com/milvus-io/milvus/pkg/log" - "github.com/milvus-io/milvus/pkg/util/typeutil" -) - -type collAliasDb struct { - db *gorm.DB -} - -func (s *collAliasDb) Insert(in []*dbmodel.CollectionAlias) error { - err := s.db.Clauses(clause.OnConflict{ - // constraint UNIQUE (tenant_id, collection_alias, ts) - DoNothing: true, - }).Create(&in).Error - - if err != nil { - log.Error("insert collection alias failed", zap.Error(err)) - return err - } - - return nil -} - -func (s *collAliasDb) GetCollectionIDByAlias(tenantID string, alias string, ts typeutil.Timestamp) (typeutil.UniqueID, error) { - var r dbmodel.CollectionAlias - - err := s.db.Model(&dbmodel.CollectionAlias{}).Select("collection_id").Where("tenant_id = ? AND collection_alias = ? AND ts <= ?", tenantID, alias, ts).Order("ts desc").Take(&r).Error - - if errors.Is(err, gorm.ErrRecordNotFound) { - return 0, fmt.Errorf("get collection_id by alias not found, alias=%s, ts=%d", alias, ts) - } - if err != nil { - log.Error("get collection_id by alias failed", zap.String("tenant", tenantID), zap.String("alias", alias), zap.Uint64("ts", ts), zap.Error(err)) - return 0, err - } - - return r.CollectionID, nil -} - -func (s *collAliasDb) ListCollectionIDTs(tenantID string, ts typeutil.Timestamp) ([]*dbmodel.CollectionAlias, error) { - var r []*dbmodel.CollectionAlias - - err := s.db.Model(&dbmodel.CollectionAlias{}).Select("collection_id, MAX(ts) ts").Where("tenant_id = ? AND ts <= ?", tenantID, ts).Group("collection_id").Find(&r).Error - if err != nil { - log.Error("list collection_id & latest ts pairs in collection_aliases failed", zap.String("tenant", tenantID), zap.Uint64("ts", ts), zap.Error(err)) - return nil, err - } - - return r, nil -} - -func (s *collAliasDb) List(tenantID string, cidTsPairs []*dbmodel.CollectionAlias) ([]*dbmodel.CollectionAlias, error) { - var collAliases []*dbmodel.CollectionAlias - - inValues := make([][]interface{}, 0, len(cidTsPairs)) - for _, pair := range cidTsPairs { - in := []interface{}{pair.CollectionID, pair.Ts} - inValues = append(inValues, in) - } - - err := s.db.Model(&dbmodel.CollectionAlias{}).Select("collection_id, collection_alias"). - Where("tenant_id = ? AND is_deleted = false AND (collection_id, ts) IN ?", tenantID, inValues).Find(&collAliases).Error - if err != nil { - log.Error("list alias by collection_id and alias pairs failed", zap.String("tenant", tenantID), zap.Any("collIdTs", inValues), zap.Error(err)) - return nil, err - } - - return collAliases, nil -} diff --git a/internal/metastore/db/dao/collection_alias_test.go b/internal/metastore/db/dao/collection_alias_test.go deleted file mode 100644 index b61b610c04..0000000000 --- a/internal/metastore/db/dao/collection_alias_test.go +++ /dev/null @@ -1,205 +0,0 @@ -package dao - -import ( - "testing" - "time" - - "github.com/cockroachdb/errors" - - "github.com/DATA-DOG/go-sqlmock" - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - "github.com/milvus-io/milvus/pkg/util/typeutil" - "github.com/stretchr/testify/assert" - "gorm.io/gorm" -) - -func TestCollectionAlias_Insert(t *testing.T) { - var collAliases = []*dbmodel.CollectionAlias{ - { - TenantID: "", - CollectionID: collID1, - CollectionAlias: "test_alias_1", - Ts: ts, - IsDeleted: false, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - }, - } - - // expectation - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `collection_aliases` (`tenant_id`,`collection_id`,`collection_alias`,`ts`,`is_deleted`,`created_at`,`updated_at`) VALUES (?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE `id`=`id`"). - WithArgs(collAliases[0].TenantID, collAliases[0].CollectionID, collAliases[0].CollectionAlias, collAliases[0].Ts, collAliases[0].IsDeleted, collAliases[0].CreatedAt, collAliases[0].UpdatedAt). - WillReturnResult(sqlmock.NewResult(100, 2)) - mock.ExpectCommit() - - // actual - err := aliasTestDb.Insert(collAliases) - assert.NoError(t, err) -} - -func TestCollectionAlias_Insert_Error(t *testing.T) { - var collAliases = []*dbmodel.CollectionAlias{ - { - TenantID: "", - CollectionID: collID1, - CollectionAlias: "test_alias_1", - Ts: ts, - IsDeleted: false, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - }, - } - - // expectation - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `collection_aliases` (`tenant_id`,`collection_id`,`collection_alias`,`ts`,`is_deleted`,`created_at`,`updated_at`) VALUES (?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE `id`=`id`"). - WithArgs(collAliases[0].TenantID, collAliases[0].CollectionID, collAliases[0].CollectionAlias, collAliases[0].Ts, collAliases[0].IsDeleted, collAliases[0].CreatedAt, collAliases[0].UpdatedAt). - WillReturnError(errors.New("test error")) - mock.ExpectRollback() - - // actual - err := aliasTestDb.Insert(collAliases) - assert.Error(t, err) -} - -func TestCollectionAlias_GetCollectionIDByName(t *testing.T) { - alias := "test_alias_name_1" - - // expectation - mock.ExpectQuery("SELECT `collection_id` FROM `collection_aliases` WHERE tenant_id = ? AND collection_alias = ? AND ts <= ? ORDER BY ts desc LIMIT 1"). - WithArgs(tenantID, alias, ts). - WillReturnRows( - sqlmock.NewRows([]string{"collection_id"}). - AddRow(collID1)) - - // actual - res, err := aliasTestDb.GetCollectionIDByAlias(tenantID, alias, ts) - assert.NoError(t, err) - assert.Equal(t, collID1, res) -} - -func TestCollectionAlias_GetCollectionIDByName_Error(t *testing.T) { - alias := "test_alias_name_1" - - // expectation - mock.ExpectQuery("SELECT `collection_id` FROM `collection_aliases` WHERE tenant_id = ? AND collection_alias = ? AND ts <= ? ORDER BY ts desc LIMIT 1"). - WithArgs(tenantID, alias, ts). - WillReturnError(errors.New("test error")) - - // actual - res, err := aliasTestDb.GetCollectionIDByAlias(tenantID, alias, ts) - assert.Equal(t, typeutil.UniqueID(0), res) - assert.Error(t, err) -} - -func TestCollectionAlias_GetCollectionIDByName_ErrRecordNotFound(t *testing.T) { - alias := "test_alias_name_1" - - // expectation - mock.ExpectQuery("SELECT `collection_id` FROM `collection_aliases` WHERE tenant_id = ? AND collection_alias = ? AND ts <= ? ORDER BY ts desc LIMIT 1"). - WithArgs(tenantID, alias, ts). - WillReturnError(gorm.ErrRecordNotFound) - - // actual - res, err := aliasTestDb.GetCollectionIDByAlias(tenantID, alias, ts) - assert.Equal(t, typeutil.UniqueID(0), res) - assert.Error(t, err) -} - -func TestCollectionAlias_ListCidTs(t *testing.T) { - var collAliases = []*dbmodel.CollectionAlias{ - { - CollectionID: collID1, - Ts: typeutil.Timestamp(2), - }, - { - CollectionID: collID2, - Ts: typeutil.Timestamp(5), - }, - } - - // expectation - mock.ExpectQuery("SELECT collection_id, MAX(ts) ts FROM `collection_aliases` WHERE tenant_id = ? AND ts <= ? GROUP BY `collection_id`"). - WithArgs(tenantID, ts). - WillReturnRows( - sqlmock.NewRows([]string{"collection_id", "ts"}). - AddRow(collID1, typeutil.Timestamp(2)). - AddRow(collID2, typeutil.Timestamp(5))) - - // actual - res, err := aliasTestDb.ListCollectionIDTs(tenantID, ts) - assert.NoError(t, err) - assert.Equal(t, collAliases, res) -} - -func TestCollectionAlias_ListCidTs_Error(t *testing.T) { - // expectation - mock.ExpectQuery("SELECT collection_id, MAX(ts) ts FROM `collection_aliases` WHERE tenant_id = ? AND ts <= ? GROUP BY `collection_id`"). - WithArgs(tenantID, ts). - WillReturnError(errors.New("test error")) - - // actual - res, err := aliasTestDb.ListCollectionIDTs(tenantID, ts) - assert.Nil(t, res) - assert.Error(t, err) -} - -func TestCollectionAlias_List(t *testing.T) { - var cidTsPairs = []*dbmodel.CollectionAlias{ - { - CollectionID: collID1, - Ts: typeutil.Timestamp(2), - }, - { - CollectionID: collID2, - Ts: typeutil.Timestamp(5), - }, - } - var out = []*dbmodel.CollectionAlias{ - { - CollectionID: collID1, - CollectionAlias: "test_alias_1", - }, - { - CollectionID: collID2, - CollectionAlias: "test_alias_2", - }, - } - - // expectation - mock.ExpectQuery("SELECT collection_id, collection_alias FROM `collection_aliases` WHERE tenant_id = ? AND is_deleted = false AND (collection_id, ts) IN ((?,?),(?,?))"). - WithArgs(tenantID, cidTsPairs[0].CollectionID, cidTsPairs[0].Ts, cidTsPairs[1].CollectionID, cidTsPairs[1].Ts). - WillReturnRows( - sqlmock.NewRows([]string{"collection_id", "collection_alias"}). - AddRow(collID1, "test_alias_1"). - AddRow(collID2, "test_alias_2")) - - // actual - res, err := aliasTestDb.List(tenantID, cidTsPairs) - assert.NoError(t, err) - assert.Equal(t, out, res) -} - -func TestCollectionAlias_List_Error(t *testing.T) { - var cidTsPairs = []*dbmodel.CollectionAlias{ - { - CollectionID: collID1, - Ts: typeutil.Timestamp(2), - }, - { - CollectionID: collID2, - Ts: typeutil.Timestamp(5), - }, - } - - // expectation - mock.ExpectQuery("SELECT collection_id, collection_alias FROM `collection_aliases` WHERE tenant_id = ? AND is_deleted = false AND (collection_id, ts) IN ((?,?),(?,?))"). - WithArgs(tenantID, cidTsPairs[0].CollectionID, cidTsPairs[0].Ts, cidTsPairs[1].CollectionID, cidTsPairs[1].Ts). - WillReturnError(errors.New("test error")) - - // actual - res, err := aliasTestDb.List(tenantID, cidTsPairs) - assert.Nil(t, res) - assert.Error(t, err) -} diff --git a/internal/metastore/db/dao/collection_channel.go b/internal/metastore/db/dao/collection_channel.go deleted file mode 100644 index 4291fc522d..0000000000 --- a/internal/metastore/db/dao/collection_channel.go +++ /dev/null @@ -1,36 +0,0 @@ -package dao - -import ( - "go.uber.org/zap" - "gorm.io/gorm" - - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - "github.com/milvus-io/milvus/pkg/log" - "github.com/milvus-io/milvus/pkg/util/typeutil" -) - -type collChannelDb struct { - db *gorm.DB -} - -func (s *collChannelDb) GetByCollectionID(tenantID string, collectionID typeutil.UniqueID, ts typeutil.Timestamp) ([]*dbmodel.CollectionChannel, error) { - var r []*dbmodel.CollectionChannel - - err := s.db.Model(&dbmodel.CollectionChannel{}).Where("tenant_id = ? AND collection_id = ? AND ts = ? AND is_deleted = false", tenantID, collectionID, ts).Find(&r).Error - if err != nil { - log.Error("get channels by collection_id and ts failed", zap.String("tenant", tenantID), zap.Int64("collectionID", collectionID), zap.Uint64("ts", ts), zap.Error(err)) - return nil, err - } - - return r, nil -} - -func (s *collChannelDb) Insert(in []*dbmodel.CollectionChannel) error { - err := s.db.CreateInBatches(in, 100).Error - if err != nil { - log.Error("insert channel failed", zap.Error(err)) - return err - } - - return nil -} diff --git a/internal/metastore/db/dao/collection_channel_test.go b/internal/metastore/db/dao/collection_channel_test.go deleted file mode 100644 index 6e1a11d41d..0000000000 --- a/internal/metastore/db/dao/collection_channel_test.go +++ /dev/null @@ -1,106 +0,0 @@ -package dao - -import ( - "testing" - "time" - - "github.com/cockroachdb/errors" - - "github.com/DATA-DOG/go-sqlmock" - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - "github.com/stretchr/testify/assert" -) - -func TestCollectionChannel_GetByCollID(t *testing.T) { - var collChannels = []*dbmodel.CollectionChannel{ - { - TenantID: tenantID, - CollectionID: collID1, - VirtualChannelName: "test_virtual_channel_1", - PhysicalChannelName: "test_physical_channel_1", - Removed: false, - Ts: ts, - IsDeleted: false, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - }, - } - - // expectation - mock.ExpectQuery("SELECT * FROM `collection_channels` WHERE tenant_id = ? AND collection_id = ? AND ts = ? AND is_deleted = false"). - WithArgs(tenantID, collID1, ts). - WillReturnRows( - sqlmock.NewRows([]string{"tenant_id", "collection_id", "virtual_channel_name", "physical_channel_name", "removed", "ts", "is_deleted", "created_at", "updated_at"}). - AddRow(collChannels[0].TenantID, collChannels[0].CollectionID, collChannels[0].VirtualChannelName, collChannels[0].PhysicalChannelName, collChannels[0].Removed, collChannels[0].Ts, collChannels[0].IsDeleted, collChannels[0].CreatedAt, collChannels[0].UpdatedAt)) - - // actual - res, err := channelTestDb.GetByCollectionID(tenantID, collID1, ts) - assert.NoError(t, err) - assert.Equal(t, collChannels, res) -} - -func TestCollectionChannel_GetByCollID_Error(t *testing.T) { - // expectation - mock.ExpectQuery("SELECT * FROM `collection_channels` WHERE tenant_id = ? AND collection_id = ? AND ts = ? AND is_deleted = false"). - WithArgs(tenantID, collID1, ts). - WillReturnError(errors.New("test error")) - - // actual - res, err := channelTestDb.GetByCollectionID(tenantID, collID1, ts) - assert.Nil(t, res) - assert.Error(t, err) -} - -func TestCollectionChannel_Insert(t *testing.T) { - var collChannels = []*dbmodel.CollectionChannel{ - { - TenantID: "", - CollectionID: collID1, - VirtualChannelName: "test_virtual_channel_1", - PhysicalChannelName: "test_physical_channel_1", - Removed: false, - Ts: ts, - IsDeleted: false, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - }, - } - - // expectation - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `collection_channels` (`tenant_id`,`collection_id`,`virtual_channel_name`,`physical_channel_name`,`removed`,`ts`,`is_deleted`,`created_at`,`updated_at`) VALUES (?,?,?,?,?,?,?,?,?)"). - WithArgs(collChannels[0].TenantID, collChannels[0].CollectionID, collChannels[0].VirtualChannelName, collChannels[0].PhysicalChannelName, collChannels[0].Removed, collChannels[0].Ts, collChannels[0].IsDeleted, collChannels[0].CreatedAt, collChannels[0].UpdatedAt). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - - // actual - err := channelTestDb.Insert(collChannels) - assert.NoError(t, err) -} - -func TestCollectionChannel_Insert_Error(t *testing.T) { - var collChannels = []*dbmodel.CollectionChannel{ - { - TenantID: "", - CollectionID: collID1, - VirtualChannelName: "test_virtual_channel_1", - PhysicalChannelName: "test_physical_channel_1", - Removed: false, - Ts: ts, - IsDeleted: false, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - }, - } - - // expectation - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `collection_channels` (`tenant_id`,`collection_id`,`virtual_channel_name`,`physical_channel_name`,`removed`,`ts`,`is_deleted`,`created_at`,`updated_at`) VALUES (?,?,?,?,?,?,?,?,?)"). - WithArgs(collChannels[0].TenantID, collChannels[0].CollectionID, collChannels[0].VirtualChannelName, collChannels[0].PhysicalChannelName, collChannels[0].Removed, collChannels[0].Ts, collChannels[0].IsDeleted, collChannels[0].CreatedAt, collChannels[0].UpdatedAt). - WillReturnError(errors.New("test error")) - mock.ExpectRollback() - - // actual - err := channelTestDb.Insert(collChannels) - assert.Error(t, err) -} diff --git a/internal/metastore/db/dao/collection_test.go b/internal/metastore/db/dao/collection_test.go deleted file mode 100644 index b2786b70cf..0000000000 --- a/internal/metastore/db/dao/collection_test.go +++ /dev/null @@ -1,495 +0,0 @@ -package dao - -import ( - "context" - "database/sql" - "database/sql/driver" - "os" - "testing" - "time" - - "github.com/cockroachdb/errors" - - "github.com/milvus-io/milvus/pkg/common" - - "github.com/DATA-DOG/go-sqlmock" - commonpb "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" - "github.com/milvus-io/milvus/internal/metastore/db/dbcore" - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - "github.com/milvus-io/milvus/pkg/util/typeutil" - "github.com/stretchr/testify/assert" - "gorm.io/driver/mysql" - "gorm.io/gorm" -) - -const ( - tenantID = "test_tenant" - noTs = typeutil.Timestamp(0) - ts = typeutil.Timestamp(10) - collID1 = typeutil.UniqueID(101) - collID2 = typeutil.UniqueID(102) - fieldID1 = typeutil.UniqueID(501) - indexID1 = typeutil.UniqueID(1001) - indexID2 = typeutil.UniqueID(1002) - segmentID1 = typeutil.UniqueID(2001) - segmentID2 = typeutil.UniqueID(2002) - partitionID1 = typeutil.UniqueID(3001) - indexBuildID1 = typeutil.UniqueID(5001) - NumRows = 1025 -) - -var ( - mock sqlmock.Sqlmock - collTestDb dbmodel.ICollectionDb - aliasTestDb dbmodel.ICollAliasDb - channelTestDb dbmodel.ICollChannelDb - fieldTestDb dbmodel.IFieldDb - partitionTestDb dbmodel.IPartitionDb - indexTestDb dbmodel.IIndexDb - segIndexTestDb dbmodel.ISegmentIndexDb - userTestDb dbmodel.IUserDb - roleTestDb dbmodel.IRoleDb - userRoleTestDb dbmodel.IUserRoleDb - grantTestDb dbmodel.IGrantDb - grantIDTestDb dbmodel.IGrantIDDb - - properties = []*commonpb.KeyValuePair{ - { - Key: common.CollectionTTLConfigKey, - Value: "3600", - }, - } - propertiesStr, _ = dbmodel.MarshalProperties(properties) -) - -// TestMain is the first function executed in current package, we will do some initial here -func TestMain(m *testing.M) { - var ( - db *sql.DB - err error - ctx = context.TODO() - ) - - // setting sql MUST exact match - db, mock, err = sqlmock.New(sqlmock.QueryMatcherOption(sqlmock.QueryMatcherEqual)) - if err != nil { - panic(err) - } - - DB, err := gorm.Open(mysql.New(mysql.Config{ - Conn: db, - SkipInitializeWithVersion: true, - }), &gorm.Config{}) - if err != nil { - panic(err) - } - - // set mocked database - dbcore.SetGlobalDB(DB) - - collTestDb = NewMetaDomain().CollectionDb(ctx) - aliasTestDb = NewMetaDomain().CollAliasDb(ctx) - channelTestDb = NewMetaDomain().CollChannelDb(ctx) - fieldTestDb = NewMetaDomain().FieldDb(ctx) - partitionTestDb = NewMetaDomain().PartitionDb(ctx) - indexTestDb = NewMetaDomain().IndexDb(ctx) - segIndexTestDb = NewMetaDomain().SegmentIndexDb(ctx) - userTestDb = NewMetaDomain().UserDb(ctx) - roleTestDb = NewMetaDomain().RoleDb(ctx) - userRoleTestDb = NewMetaDomain().UserRoleDb(ctx) - grantTestDb = NewMetaDomain().GrantDb(ctx) - grantIDTestDb = NewMetaDomain().GrantIDDb(ctx) - - // m.Run entry for executing tests - os.Exit(m.Run()) -} - -// Notice: sql must be exactly matched, we can use debug() to print the sql - -func TestCollection_GetCidTs_Ts0(t *testing.T) { - var collection = &dbmodel.Collection{ - CollectionID: collID1, - Ts: noTs, - } - - // expectation - mock.ExpectQuery("SELECT collection_id, ts FROM `collections` WHERE tenant_id = ? AND collection_id = ? AND ts <= ? ORDER BY ts desc LIMIT 1"). - WithArgs(tenantID, collID1, noTs). - WillReturnRows( - sqlmock.NewRows([]string{"collection_id", "ts"}). - AddRow(collID1, noTs)) - - // actual - res, err := collTestDb.GetCollectionIDTs(tenantID, collID1, noTs) - assert.NoError(t, err) - assert.Equal(t, collection, res) -} - -func TestCollection_GetCidTs_TsNot0(t *testing.T) { - resultTs := typeutil.Timestamp(2) - var collection = &dbmodel.Collection{ - CollectionID: collID1, - Ts: resultTs, - } - - // expectation - mock.ExpectQuery("SELECT collection_id, ts FROM `collections` WHERE tenant_id = ? AND collection_id = ? AND ts <= ? ORDER BY ts desc LIMIT 1"). - WithArgs(tenantID, collID1, ts). - WillReturnRows( - sqlmock.NewRows([]string{"collection_id", "ts"}). - AddRow(collID1, resultTs)) - - // actual - res, err := collTestDb.GetCollectionIDTs(tenantID, collID1, ts) - assert.NoError(t, err) - assert.Equal(t, collection, res) -} - -func TestCollection_GetCidTs_TsNot0_Error(t *testing.T) { - // expectation - mock.ExpectQuery("SELECT collection_id, ts FROM `collections` WHERE tenant_id = ? AND collection_id = ? AND ts <= ? ORDER BY ts desc LIMIT 1"). - WithArgs(tenantID, collID1, ts). - WillReturnError(errors.New("test error")) - - // actual - res, err := collTestDb.GetCollectionIDTs(tenantID, collID1, ts) - assert.Nil(t, res) - assert.Error(t, err) -} - -func TestCollection_GetCidTs_TsNot0_ErrRecordNotFound(t *testing.T) { - // expectation - mock.ExpectQuery("SELECT collection_id, ts FROM `collections` WHERE tenant_id = ? AND collection_id = ? AND ts <= ? ORDER BY ts desc LIMIT 1"). - WithArgs(tenantID, collID1, ts). - WillReturnError(gorm.ErrRecordNotFound) - - // actual - res, err := collTestDb.GetCollectionIDTs(tenantID, collID1, ts) - assert.Nil(t, res) - assert.Error(t, err) -} - -func TestCollection_ListCidTs_TsNot0(t *testing.T) { - var collection = []*dbmodel.Collection{ - { - CollectionID: collID1, - Ts: typeutil.Timestamp(2), - }, - { - CollectionID: collID2, - Ts: typeutil.Timestamp(5), - }, - } - - // expectation - mock.ExpectQuery("SELECT collection_id, MAX(ts) ts FROM `collections` WHERE tenant_id = ? AND ts <= ? GROUP BY `collection_id`"). - WithArgs(tenantID, ts). - WillReturnRows( - sqlmock.NewRows([]string{"collection_id", "ts"}). - AddRow(collID1, typeutil.Timestamp(2)). - AddRow(collID2, typeutil.Timestamp(5))) - - // actual - res, err := collTestDb.ListCollectionIDTs(tenantID, ts) - assert.NoError(t, err) - assert.Equal(t, collection, res) -} - -func TestCollection_ListCidTs_TsNot0_Error(t *testing.T) { - // expectation - mock.ExpectQuery("SELECT collection_id, MAX(ts) ts FROM `collections` WHERE tenant_id = ? AND ts <= ? GROUP BY `collection_id`"). - WithArgs(tenantID, ts). - WillReturnError(errors.New("test error")) - - // actual - res, err := collTestDb.ListCollectionIDTs(tenantID, ts) - assert.Nil(t, res) - assert.Error(t, err) -} - -func TestCollection_ListCidTs_Ts0(t *testing.T) { - var collection = []*dbmodel.Collection{ - { - CollectionID: collID1, - Ts: noTs, - }, - { - CollectionID: collID2, - Ts: noTs, - }, - } - - // expectation - mock.ExpectQuery("SELECT collection_id, MAX(ts) ts FROM `collections` WHERE tenant_id = ? AND ts <= ? GROUP BY `collection_id`"). - WithArgs(tenantID, noTs). - WillReturnRows( - sqlmock.NewRows([]string{"collection_id", "ts"}). - AddRow(collID1, noTs). - AddRow(collID2, noTs)) - - // actual - res, err := collTestDb.ListCollectionIDTs(tenantID, noTs) - assert.NoError(t, err) - assert.Equal(t, collection, res) -} - -func TestCollection_Get(t *testing.T) { - var collection = &dbmodel.Collection{ - TenantID: "", - CollectionID: collID1, - CollectionName: "test_collection_name_1", - Description: "", - AutoID: false, - ShardsNum: common.DefaultShardsNum, - StartPosition: "", - ConsistencyLevel: int32(commonpb.ConsistencyLevel_Eventually), - Ts: ts, - } - - // expectation - mock.ExpectQuery("SELECT * FROM `collections` WHERE tenant_id = ? AND collection_id = ? AND ts = ? AND is_deleted = false LIMIT 1"). - WithArgs(tenantID, collection.CollectionID, collection.Ts). - WillReturnRows( - sqlmock.NewRows([]string{"tenant_id", "collection_id", "collection_name", "description", "auto_id", "shards_num", "start_position", "consistency_level", "ts"}). - AddRow(collection.TenantID, collection.CollectionID, collection.CollectionName, collection.Description, collection.AutoID, collection.ShardsNum, collection.StartPosition, collection.ConsistencyLevel, collection.Ts)) - - // actual - res, err := collTestDb.Get(tenantID, collID1, ts) - assert.NoError(t, err) - assert.Equal(t, collection, res) -} - -func TestCollection_Get_Error(t *testing.T) { - var collection = &dbmodel.Collection{ - TenantID: "", - CollectionID: collID1, - CollectionName: "test_collection_name_1", - Description: "", - AutoID: false, - ShardsNum: common.DefaultShardsNum, - StartPosition: "", - ConsistencyLevel: int32(commonpb.ConsistencyLevel_Eventually), - Ts: ts, - } - - // expectation - mock.ExpectQuery("SELECT * FROM `collections` WHERE tenant_id = ? AND collection_id = ? AND ts = ? AND is_deleted = false LIMIT 1"). - WithArgs(tenantID, collection.CollectionID, collection.Ts). - WillReturnError(errors.New("test error")) - - // actual - res, err := collTestDb.Get(tenantID, collID1, ts) - assert.Nil(t, res) - assert.Error(t, err) -} - -func TestCollection_Get_ErrRecordNotFound(t *testing.T) { - var collection = &dbmodel.Collection{ - TenantID: "", - CollectionID: collID1, - CollectionName: "test_collection_name_1", - Description: "", - AutoID: false, - ShardsNum: common.DefaultShardsNum, - StartPosition: "", - ConsistencyLevel: int32(commonpb.ConsistencyLevel_Eventually), - Ts: ts, - } - - // expectation - mock.ExpectQuery("SELECT * FROM `collections` WHERE tenant_id = ? AND collection_id = ? AND ts = ? AND is_deleted = false LIMIT 1"). - WithArgs(tenantID, collection.CollectionID, collection.Ts). - WillReturnError(gorm.ErrRecordNotFound) - - // actual - res, err := collTestDb.Get(tenantID, collID1, ts) - assert.Nil(t, res) - assert.Error(t, err) -} - -func TestCollection_GetCollectionIDByName(t *testing.T) { - collectionName := "test_collection_name_1" - - // expectation - mock.ExpectQuery("SELECT `collection_id` FROM `collections` WHERE tenant_id = ? AND collection_name = ? AND ts <= ? ORDER BY ts desc LIMIT 1"). - WithArgs(tenantID, collectionName, ts). - WillReturnRows( - sqlmock.NewRows([]string{"collection_id"}). - AddRow(collID1)) - - // actual - res, err := collTestDb.GetCollectionIDByName(tenantID, collectionName, ts) - assert.NoError(t, err) - assert.Equal(t, collID1, res) -} - -func TestCollection_GetCollectionIDByName_Error(t *testing.T) { - collectionName := "test_collection_name_1" - - // expectation - mock.ExpectQuery("SELECT `collection_id` FROM `collections` WHERE tenant_id = ? AND collection_name = ? AND ts <= ? ORDER BY ts desc LIMIT 1"). - WithArgs(tenantID, collectionName, ts). - WillReturnError(errors.New("test error")) - - // actual - res, err := collTestDb.GetCollectionIDByName(tenantID, collectionName, ts) - assert.Equal(t, typeutil.UniqueID(0), res) - assert.Error(t, err) -} - -func TestCollection_GetCollectionIDByName_ErrRecordNotFound(t *testing.T) { - collectionName := "test_collection_name_1" - - // expectation - mock.ExpectQuery("SELECT `collection_id` FROM `collections` WHERE tenant_id = ? AND collection_name = ? AND ts <= ? ORDER BY ts desc LIMIT 1"). - WithArgs(tenantID, collectionName, ts). - WillReturnError(gorm.ErrRecordNotFound) - - // actual - res, err := collTestDb.GetCollectionIDByName(tenantID, collectionName, ts) - assert.Equal(t, typeutil.UniqueID(0), res) - assert.Error(t, err) -} - -func TestCollection_Insert(t *testing.T) { - var collection = &dbmodel.Collection{ - TenantID: "", - CollectionID: collID1, - CollectionName: "test_collection_name_1", - Description: "", - AutoID: false, - ShardsNum: common.DefaultShardsNum, - StartPosition: "", - ConsistencyLevel: int32(commonpb.ConsistencyLevel_Eventually), - Properties: propertiesStr, - Ts: ts, - IsDeleted: false, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - } - - // expectation - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `collections` (`tenant_id`,`collection_id`,`collection_name`,`description`,`auto_id`,`shards_num`,`start_position`,`consistency_level`,`status`,`properties`,`ts`,`is_deleted`,`created_at`,`updated_at`) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE `id`=`id`"). - WithArgs(collection.TenantID, collection.CollectionID, collection.CollectionName, collection.Description, collection.AutoID, collection.ShardsNum, collection.StartPosition, collection.ConsistencyLevel, collection.Status, collection.Properties, collection.Ts, collection.IsDeleted, collection.CreatedAt, collection.UpdatedAt). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - - // actual - err := collTestDb.Insert(collection) - assert.NoError(t, err) -} - -func TestCollection_Insert_Error(t *testing.T) { - var collection = &dbmodel.Collection{ - TenantID: "", - CollectionID: collID1, - CollectionName: "test_collection_name_1", - Description: "", - AutoID: false, - ShardsNum: common.DefaultShardsNum, - StartPosition: "", - ConsistencyLevel: int32(commonpb.ConsistencyLevel_Eventually), - Ts: ts, - IsDeleted: false, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - } - - // expectation - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `collections` (`tenant_id`,`collection_id`,`collection_name`,`description`,`auto_id`,`shards_num`,`start_position`,`consistency_level`,`status`,`properties`,`ts`,`is_deleted`,`created_at`,`updated_at`) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?) ON DUPLICATE KEY UPDATE `id`=`id`"). - WithArgs(collection.TenantID, collection.CollectionID, collection.CollectionName, collection.Description, collection.AutoID, collection.ShardsNum, collection.StartPosition, collection.ConsistencyLevel, collection.Status, collection.Properties, collection.Ts, collection.IsDeleted, collection.CreatedAt, collection.UpdatedAt). - WillReturnError(errors.New("test error")) - mock.ExpectRollback() - - // actual - err := collTestDb.Insert(collection) - assert.Error(t, err) -} - -type AnyTime struct{} - -func (a AnyTime) Match(v driver.Value) bool { - _, ok := v.(time.Time) - return ok -} - -func GetBase() dbmodel.Base { - return dbmodel.Base{ - TenantID: tenantID, - IsDeleted: false, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - } -} - -func SuccessExec(f func()) { - mock.ExpectBegin() - f() - mock.ExpectCommit() -} - -func ErrorExec(f func()) { - mock.ExpectBegin() - f() - mock.ExpectRollback() -} - -func Test_collectionDb_Update(t *testing.T) { - t.Run("normal case", func(t *testing.T) { - var collection = &dbmodel.Collection{ - TenantID: "", - CollectionID: collID1, - CollectionName: "test_collection_name_1", - Description: "", - AutoID: false, - ShardsNum: common.DefaultShardsNum, - StartPosition: "", - ConsistencyLevel: int32(commonpb.ConsistencyLevel_Eventually), - Ts: ts, - IsDeleted: false, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - } - - // expectation - mock.ExpectBegin() - mock.ExpectExec("UPDATE `collections` SET `auto_id`=?,`collection_id`=?,`collection_name`=?,`consistency_level`=?,`created_at`=?,`description`=?,`is_deleted`=?,`properties`=?,`shards_num`=?,`start_position`=?,`status`=?,`tenant_id`=?,`ts`=?,`updated_at`=? WHERE id = ?"). - WithArgs(collection.AutoID, collection.CollectionID, collection.CollectionName, collection.ConsistencyLevel, collection.CreatedAt, collection.Description, collection.IsDeleted, collection.Properties, collection.ShardsNum, collection.StartPosition, collection.Status, collection.TenantID, collection.Ts, collection.UpdatedAt, collection.ID). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - - // actual - err := collTestDb.Update(collection) - assert.NoError(t, err) - }) - - t.Run("error", func(t *testing.T) { - var collection = &dbmodel.Collection{ - TenantID: "", - CollectionID: collID1, - CollectionName: "test_collection_name_1", - Description: "", - AutoID: false, - ShardsNum: common.DefaultShardsNum, - StartPosition: "", - ConsistencyLevel: int32(commonpb.ConsistencyLevel_Eventually), - Ts: ts, - IsDeleted: false, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - } - - // expectation - mock.ExpectBegin() - mock.ExpectExec("UPDATE `collections` SET `auto_id`=?,`collection_id`=?,`collection_name`=?,`consistency_level`=?,`created_at`=?,`description`=?,`is_deleted`=?,`properties`=?,`shards_num`=?,`start_position`=?,`status`=?,`tenant_id`=?,`ts`=?,`updated_at`=? WHERE id = ?"). - WithArgs(collection.AutoID, collection.CollectionID, collection.CollectionName, collection.ConsistencyLevel, collection.CreatedAt, collection.Description, collection.IsDeleted, collection.Properties, collection.ShardsNum, collection.StartPosition, collection.Status, collection.TenantID, collection.Ts, collection.UpdatedAt, collection.ID). - WillReturnError(errors.New("error mock Update")) - mock.ExpectRollback() - - // actual - err := collTestDb.Update(collection) - assert.Error(t, err) - }) -} diff --git a/internal/metastore/db/dao/common.go b/internal/metastore/db/dao/common.go deleted file mode 100644 index 06724e14bc..0000000000 --- a/internal/metastore/db/dao/common.go +++ /dev/null @@ -1,62 +0,0 @@ -package dao - -import ( - "context" - - "github.com/milvus-io/milvus/internal/metastore/db/dbcore" - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" -) - -type metaDomain struct{} - -func NewMetaDomain() *metaDomain { - return &metaDomain{} -} - -func (*metaDomain) CollectionDb(ctx context.Context) dbmodel.ICollectionDb { - return &collectionDb{dbcore.GetDB(ctx)} -} - -func (*metaDomain) FieldDb(ctx context.Context) dbmodel.IFieldDb { - return &fieldDb{dbcore.GetDB(ctx)} -} - -func (*metaDomain) CollChannelDb(ctx context.Context) dbmodel.ICollChannelDb { - return &collChannelDb{dbcore.GetDB(ctx)} -} - -func (*metaDomain) CollAliasDb(ctx context.Context) dbmodel.ICollAliasDb { - return &collAliasDb{dbcore.GetDB(ctx)} -} - -func (*metaDomain) PartitionDb(ctx context.Context) dbmodel.IPartitionDb { - return &partitionDb{dbcore.GetDB(ctx)} -} - -func (*metaDomain) IndexDb(ctx context.Context) dbmodel.IIndexDb { - return &indexDb{dbcore.GetDB(ctx)} -} - -func (*metaDomain) SegmentIndexDb(ctx context.Context) dbmodel.ISegmentIndexDb { - return &segmentIndexDb{dbcore.GetDB(ctx)} -} - -func (*metaDomain) UserDb(ctx context.Context) dbmodel.IUserDb { - return &userDb{dbcore.GetDB(ctx)} -} - -func (d *metaDomain) RoleDb(ctx context.Context) dbmodel.IRoleDb { - return &roleDb{dbcore.GetDB(ctx)} -} - -func (d *metaDomain) UserRoleDb(ctx context.Context) dbmodel.IUserRoleDb { - return &userRoleDb{dbcore.GetDB(ctx)} -} - -func (d *metaDomain) GrantDb(ctx context.Context) dbmodel.IGrantDb { - return &grantDb{dbcore.GetDB(ctx)} -} - -func (d *metaDomain) GrantIDDb(ctx context.Context) dbmodel.IGrantIDDb { - return &grantIDDb{dbcore.GetDB(ctx)} -} diff --git a/internal/metastore/db/dao/field.go b/internal/metastore/db/dao/field.go deleted file mode 100644 index c69b602032..0000000000 --- a/internal/metastore/db/dao/field.go +++ /dev/null @@ -1,36 +0,0 @@ -package dao - -import ( - "go.uber.org/zap" - "gorm.io/gorm" - - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - "github.com/milvus-io/milvus/pkg/log" - "github.com/milvus-io/milvus/pkg/util/typeutil" -) - -type fieldDb struct { - db *gorm.DB -} - -func (s *fieldDb) GetByCollectionID(tenantID string, collectionID typeutil.UniqueID, ts typeutil.Timestamp) ([]*dbmodel.Field, error) { - var r []*dbmodel.Field - - err := s.db.Model(&dbmodel.Field{}).Where("tenant_id = ? AND collection_id = ? AND ts = ? AND is_deleted = false", tenantID, collectionID, ts).Find(&r).Error - if err != nil { - log.Error("get fields by collection_id and ts failed", zap.String("tenant", tenantID), zap.Int64("collectionID", collectionID), zap.Uint64("ts", ts), zap.Error(err)) - return nil, err - } - - return r, nil -} - -func (s *fieldDb) Insert(in []*dbmodel.Field) error { - err := s.db.CreateInBatches(in, 100).Error - if err != nil { - log.Error("insert field failed", zap.Error(err)) - return err - } - - return nil -} diff --git a/internal/metastore/db/dao/field_test.go b/internal/metastore/db/dao/field_test.go deleted file mode 100644 index b6924c95f4..0000000000 --- a/internal/metastore/db/dao/field_test.go +++ /dev/null @@ -1,119 +0,0 @@ -package dao - -import ( - "testing" - "time" - - "github.com/cockroachdb/errors" - - "github.com/DATA-DOG/go-sqlmock" - "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - "github.com/stretchr/testify/assert" -) - -func TestField_GetByCollID(t *testing.T) { - var fields = []*dbmodel.Field{ - { - TenantID: tenantID, - FieldID: fieldID1, - FieldName: "test_field_1", - IsPrimaryKey: false, - Description: "", - DataType: schemapb.DataType_FloatVector, - TypeParams: "", - IndexParams: "", - AutoID: false, - CollectionID: collID1, - Ts: ts, - }, - } - - // expectation - mock.ExpectQuery("SELECT * FROM `field_schemas` WHERE tenant_id = ? AND collection_id = ? AND ts = ? AND is_deleted = false"). - WithArgs(tenantID, collID1, ts). - WillReturnRows( - sqlmock.NewRows([]string{"tenant_id", "field_id", "field_name", "is_primary_key", "description", "data_type", "type_params", "index_params", "auto_id", "collection_id", "ts"}). - AddRow(fields[0].TenantID, fields[0].FieldID, fields[0].FieldName, fields[0].IsPrimaryKey, fields[0].Description, fields[0].DataType, fields[0].TypeParams, fields[0].IndexParams, fields[0].AutoID, fields[0].CollectionID, fields[0].Ts)) - - // actual - res, err := fieldTestDb.GetByCollectionID(tenantID, collID1, ts) - assert.NoError(t, err) - assert.Equal(t, fields, res) -} - -func TestField_GetByCollID_Error(t *testing.T) { - // expectation - mock.ExpectQuery("SELECT * FROM `field_schemas` WHERE tenant_id = ? AND collection_id = ? AND ts = ? AND is_deleted = false"). - WithArgs(tenantID, collID1, ts). - WillReturnError(errors.New("test error")) - - // actual - res, err := fieldTestDb.GetByCollectionID(tenantID, collID1, ts) - assert.Nil(t, res) - assert.Error(t, err) -} - -func TestField_Insert(t *testing.T) { - var fields = []*dbmodel.Field{ - { - TenantID: tenantID, - FieldID: fieldID1, - FieldName: "test_field_1", - IsPrimaryKey: false, - Description: "", - DataType: schemapb.DataType_FloatVector, - TypeParams: "", - IndexParams: "", - AutoID: false, - CollectionID: collID1, - Ts: ts, - IsDeleted: false, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - }, - } - - // expectation - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `field_schemas` (`tenant_id`,`field_id`,`field_name`,`is_primary_key`,`description`,`data_type`,`type_params`,`index_params`,`auto_id`,`collection_id`,`ts`,`is_deleted`,`created_at`,`updated_at`) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)"). - WithArgs(fields[0].TenantID, fields[0].FieldID, fields[0].FieldName, fields[0].IsPrimaryKey, fields[0].Description, fields[0].DataType, fields[0].TypeParams, fields[0].IndexParams, fields[0].AutoID, fields[0].CollectionID, fields[0].Ts, fields[0].IsDeleted, fields[0].CreatedAt, fields[0].UpdatedAt). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - - // actual - err := fieldTestDb.Insert(fields) - assert.NoError(t, err) -} - -func TestField_Insert_Error(t *testing.T) { - var fields = []*dbmodel.Field{ - { - TenantID: tenantID, - FieldID: fieldID1, - FieldName: "test_field_1", - IsPrimaryKey: false, - Description: "", - DataType: schemapb.DataType_FloatVector, - TypeParams: "", - IndexParams: "", - AutoID: false, - CollectionID: collID1, - Ts: ts, - IsDeleted: false, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - }, - } - - // expectation - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `field_schemas` (`tenant_id`,`field_id`,`field_name`,`is_primary_key`,`description`,`data_type`,`type_params`,`index_params`,`auto_id`,`collection_id`,`ts`,`is_deleted`,`created_at`,`updated_at`) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?)"). - WithArgs(fields[0].TenantID, fields[0].FieldID, fields[0].FieldName, fields[0].IsPrimaryKey, fields[0].Description, fields[0].DataType, fields[0].TypeParams, fields[0].IndexParams, fields[0].AutoID, fields[0].CollectionID, fields[0].Ts, fields[0].IsDeleted, fields[0].CreatedAt, fields[0].UpdatedAt). - WillReturnError(errors.New("test error")) - mock.ExpectRollback() - - // actual - err := fieldTestDb.Insert(fields) - assert.Error(t, err) -} diff --git a/internal/metastore/db/dao/gran_id.go b/internal/metastore/db/dao/gran_id.go deleted file mode 100644 index 1704083e81..0000000000 --- a/internal/metastore/db/dao/gran_id.go +++ /dev/null @@ -1,55 +0,0 @@ -package dao - -import ( - "go.uber.org/zap" - "gorm.io/gorm" - - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - "github.com/milvus-io/milvus/pkg/log" -) - -type grantIDDb struct { - db *gorm.DB -} - -func (g *grantIDDb) GetGrantIDs(tenantID string, grantID int64, privilege string, preloadGrant bool, preloadGrantor bool) ([]*dbmodel.GrantID, error) { - var ( - grantIDs []*dbmodel.GrantID - db *gorm.DB - err error - ) - db = g.db.Model(&dbmodel.GrantID{}). - Where(&dbmodel.GrantID{GrantID: grantID, Privilege: privilege}). - Where(dbmodel.GetCommonCondition(tenantID, false)) - if preloadGrant { - db = db.Preload("Grant") - } - if preloadGrantor { - db = db.Preload("Grantor") - } - err = db.Find(&grantIDs).Error - if err != nil { - log.Error("fail to get grant ids", zap.String("tenant_id", tenantID), zap.Int64("grantID", grantID), zap.String("privilege", privilege), zap.Error(err)) - return nil, err - } - return grantIDs, err -} - -func (g *grantIDDb) Insert(in *dbmodel.GrantID) error { - err := g.db.Create(in).Error - if err != nil { - log.Error("fail to insert the grant-id", zap.Any("in", in), zap.Error(err)) - } - return err -} - -func (g *grantIDDb) Delete(tenantID string, grantID int64, privilege string) error { - err := g.db.Model(dbmodel.GrantID{}). - Where(&dbmodel.GrantID{GrantID: grantID, Privilege: privilege}). - Where(dbmodel.GetCommonCondition(tenantID, false)). - Update("is_deleted", true).Error - if err != nil { - log.Error("fail to delete the user-role", zap.String("tenant_id", tenantID), zap.Int64("grantID", grantID), zap.String("privilege", privilege), zap.Error(err)) - } - return err -} diff --git a/internal/metastore/db/dao/gran_id_test.go b/internal/metastore/db/dao/gran_id_test.go deleted file mode 100644 index 995df4c76f..0000000000 --- a/internal/metastore/db/dao/gran_id_test.go +++ /dev/null @@ -1,236 +0,0 @@ -package dao - -import ( - "testing" - - "github.com/cockroachdb/errors" - - "github.com/DATA-DOG/go-sqlmock" - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - "github.com/stretchr/testify/assert" -) - -func TestGrantID_GetGrantIDs(t *testing.T) { - var ( - grantID1 int64 = 10 - grantID2 int64 = 20 - grantorID1 int64 = 1 - grantorID2 int64 = 2 - privilege1 = "PrivilegeLoad" - privilege2 = "PrivilegeInsert" - grantIDs []*dbmodel.GrantID - err error - ) - - mock.ExpectQuery("SELECT * FROM `grant_id` WHERE `is_deleted` = ? AND `tenant_id` = ?"). - WithArgs(false, tenantID). - WillReturnRows( - sqlmock.NewRows([]string{"tenant_id", "grant_id", "grantor_id", "privilege"}). - AddRow(tenantID, grantID1, grantorID1, privilege1). - AddRow(tenantID, grantID2, grantorID2, privilege2)) - - grantIDs, err = grantIDTestDb.GetGrantIDs(tenantID, 0, "", false, false) - mock.MatchExpectationsInOrder(false) - assert.NoError(t, err) - assert.Equal(t, 2, len(grantIDs)) - assert.Equal(t, grantID1, grantIDs[0].GrantID) - assert.Equal(t, grantorID2, grantIDs[1].GrantorID) - assert.Equal(t, privilege2, grantIDs[1].Privilege) - - mock.ExpectQuery("SELECT * FROM `grant_id` WHERE `is_deleted` = ? AND `tenant_id` = ?"). - WithArgs(false, tenantID). - WillReturnError(errors.New("test error")) - _, err = grantIDTestDb.GetGrantIDs(tenantID, 0, "", false, false) - mock.MatchExpectationsInOrder(false) - assert.Error(t, err) -} - -func TestGrantID_GetGrantIDs_Preload(t *testing.T) { - var ( - grantID1 int64 = 10 - grantID2 int64 = 20 - grantorID1 int64 = 1 - grantorID2 int64 = 2 - privilege1 = "PrivilegeLoad" - privilege2 = "PrivilegeInsert" - grantIDs []*dbmodel.GrantID - err error - ) - - mock.ExpectQuery("SELECT * FROM `grant_id` WHERE `is_deleted` = ? AND `tenant_id` = ?"). - WithArgs(false, tenantID). - WillReturnRows( - sqlmock.NewRows([]string{"tenant_id", "grant_id", "grantor_id", "privilege"}). - AddRow(tenantID, grantID1, grantorID1, privilege1). - AddRow(tenantID, grantID2, grantorID2, privilege2)) - - mock.ExpectQuery("SELECT * FROM `grant` WHERE `grant`.`id` IN (?,?)"). - WithArgs(grantID1, grantID2). - WillReturnRows( - sqlmock.NewRows([]string{"id", "tenant_id", "object"}). - AddRow(grantID1, tenantID, "obj1"). - AddRow(grantID2, tenantID, "obj2")) - - mock.ExpectQuery("SELECT * FROM `credential_users` WHERE `credential_users`.`id` IN (?,?)"). - WithArgs(grantorID1, grantorID2). - WillReturnRows( - sqlmock.NewRows([]string{"id", "tenant_id", "username"}). - AddRow(grantorID1, tenantID, "fo1"). - AddRow(grantorID2, tenantID, "fo2")) - - grantIDs, err = grantIDTestDb.GetGrantIDs(tenantID, 0, "", true, true) - mock.MatchExpectationsInOrder(false) - assert.NoError(t, err) - assert.Equal(t, 2, len(grantIDs)) - assert.Equal(t, grantID1, grantIDs[0].GrantID) - assert.Equal(t, "obj1", grantIDs[0].Grant.Object) - assert.Equal(t, grantorID2, grantIDs[1].GrantorID) - assert.Equal(t, privilege2, grantIDs[1].Privilege) - assert.Equal(t, "fo2", grantIDs[1].Grantor.Username) - - mock.ExpectQuery("SELECT * FROM `grant_id` WHERE `is_deleted` = ? AND `tenant_id` = ?"). - WithArgs(false, tenantID). - WillReturnError(errors.New("test error")) - _, err = grantIDTestDb.GetGrantIDs(tenantID, 0, "", true, true) - mock.MatchExpectationsInOrder(false) - assert.Error(t, err) -} - -func TestGrantID_GetGrantIDs_WithGrant(t *testing.T) { - var ( - grantID1 int64 = 10 - grantorID1 int64 = 1 - grantorID2 int64 = 2 - privilege1 = "PrivilegeLoad" - privilege2 = "PrivilegeInsert" - grantIDs []*dbmodel.GrantID - err error - ) - - mock.ExpectQuery("SELECT * FROM `grant_id` WHERE `grant_id`.`grant_id` = ? AND `is_deleted` = ? AND `tenant_id` = ?"). - WithArgs(grantID1, false, tenantID). - WillReturnRows( - sqlmock.NewRows([]string{"tenant_id", "grant_id", "grantor_id", "privilege"}). - AddRow(tenantID, grantID1, grantorID1, privilege1). - AddRow(tenantID, grantID1, grantorID2, privilege2)) - - grantIDs, err = grantIDTestDb.GetGrantIDs(tenantID, grantID1, "", false, false) - mock.MatchExpectationsInOrder(false) - assert.NoError(t, err) - assert.Equal(t, 2, len(grantIDs)) - assert.Equal(t, grantID1, grantIDs[0].GrantID) - assert.Equal(t, grantorID2, grantIDs[1].GrantorID) - assert.Equal(t, privilege2, grantIDs[1].Privilege) -} - -func TestGrantID_GetGrantIDs_WithGrantAndPrivilege(t *testing.T) { - var ( - grantID1 int64 = 10 - grantorID1 int64 = 1 - privilege1 = "PrivilegeLoad" - grantIDs []*dbmodel.GrantID - err error - ) - - mock.ExpectQuery("SELECT * FROM `grant_id` WHERE `grant_id`.`grant_id` = ? AND `grant_id`.`privilege` = ? AND `is_deleted` = ? AND `tenant_id` = ?"). - WithArgs(grantID1, privilege1, false, tenantID). - WillReturnRows( - sqlmock.NewRows([]string{"tenant_id", "grant_id", "grantor_id", "privilege"}). - AddRow(tenantID, grantID1, grantorID1, privilege1)) - - grantIDs, err = grantIDTestDb.GetGrantIDs(tenantID, grantID1, privilege1, false, false) - mock.MatchExpectationsInOrder(false) - assert.NoError(t, err) - assert.Equal(t, 1, len(grantIDs)) - assert.Equal(t, grantID1, grantIDs[0].GrantID) - assert.Equal(t, privilege1, grantIDs[0].Privilege) -} - -func TestGrantID_Insert(t *testing.T) { - var ( - grantID *dbmodel.GrantID - err error - ) - grantID = &dbmodel.GrantID{ - Base: GetBase(), - GrantID: 1, - GrantorID: 10, - Privilege: "PrivilegeLoad", - } - - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `grant_id` (`tenant_id`,`is_deleted`,`created_at`,`updated_at`,`grant_id`,`privilege`,`grantor_id`) VALUES (?,?,?,?,?,?,?)"). - WithArgs(grantID.TenantID, grantID.IsDeleted, grantID.CreatedAt, grantID.UpdatedAt, grantID.GrantID, grantID.Privilege, grantID.GrantorID). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - err = grantIDTestDb.Insert(grantID) - mock.MatchExpectationsInOrder(false) - assert.NoError(t, err) - -} - -func TestGrantID_Insert_Error(t *testing.T) { - var ( - grantID *dbmodel.GrantID - err error - ) - grantID = &dbmodel.GrantID{ - Base: GetBase(), - GrantID: 1, - GrantorID: 10, - Privilege: "PrivilegeLoad", - } - - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `grant_id` (`tenant_id`,`is_deleted`,`created_at`,`updated_at`,`grant_id`,`privilege`,`grantor_id`) VALUES (?,?,?,?,?,?,?)"). - WithArgs(grantID.TenantID, grantID.IsDeleted, grantID.CreatedAt, grantID.UpdatedAt, grantID.GrantID, grantID.Privilege, grantID.GrantorID). - WillReturnError(errors.New("test error")) - mock.ExpectRollback() - err = grantIDTestDb.Insert(grantID) - mock.MatchExpectationsInOrder(false) - assert.Error(t, err) -} - -func TestGrantID_Delete(t *testing.T) { - var ( - grantID *dbmodel.GrantID - err error - ) - grantID = &dbmodel.GrantID{ - Base: GetBase(), - GrantID: 1, - GrantorID: 10, - Privilege: "PrivilegeLoad", - } - - mock.ExpectBegin() - mock.ExpectExec("UPDATE `grant_id` SET `is_deleted`=?,`updated_at`=? WHERE `grant_id`.`grant_id` = ? AND `grant_id`.`privilege` = ? AND `is_deleted` = ? AND `tenant_id` = ?"). - WithArgs(true, AnyTime{}, grantID.GrantID, grantID.Privilege, grantID.IsDeleted, grantID.TenantID). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - err = grantIDTestDb.Delete(grantID.TenantID, grantID.GrantID, grantID.Privilege) - mock.MatchExpectationsInOrder(false) - assert.NoError(t, err) -} - -func TestGrantID_Delete_Error(t *testing.T) { - var ( - grantID *dbmodel.GrantID - err error - ) - grantID = &dbmodel.GrantID{ - Base: GetBase(), - GrantID: 1, - GrantorID: 10, - Privilege: "PrivilegeLoad", - } - - mock.ExpectBegin() - mock.ExpectExec("UPDATE `grant_id` SET `is_deleted`=?,`updated_at`=? WHERE `grant_id`.`grant_id` = ? AND `grant_id`.`privilege` = ? AND `is_deleted` = ? AND `tenant_id` = ?"). - WithArgs(true, AnyTime{}, grantID.GrantID, grantID.Privilege, grantID.IsDeleted, grantID.TenantID). - WillReturnError(errors.New("test error")) - mock.ExpectCommit() - err = grantIDTestDb.Delete(grantID.TenantID, grantID.GrantID, grantID.Privilege) - mock.MatchExpectationsInOrder(false) - assert.Error(t, err) -} diff --git a/internal/metastore/db/dao/grant.go b/internal/metastore/db/dao/grant.go deleted file mode 100644 index 106bce10db..0000000000 --- a/internal/metastore/db/dao/grant.go +++ /dev/null @@ -1,49 +0,0 @@ -package dao - -import ( - "go.uber.org/zap" - "gorm.io/gorm" - - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - "github.com/milvus-io/milvus/pkg/log" -) - -type grantDb struct { - db *gorm.DB -} - -func (g *grantDb) GetGrants(tenantID string, roleID int64, object string, objectName string) ([]*dbmodel.Grant, error) { - var ( - grants []*dbmodel.Grant - err error - ) - err = g.db.Model(&dbmodel.Grant{}). - Where(&dbmodel.Grant{RoleID: roleID, Object: object, ObjectName: objectName}). - Where(dbmodel.GetCommonCondition(tenantID, false)). - Preload("Role"). - Find(&grants).Error - if err != nil { - log.Error("fail to get grants", zap.String("tenant_id", tenantID), zap.Int64("roleID", roleID), zap.String("object", object), zap.String("object_name", objectName), zap.Error(err)) - return nil, err - } - return grants, nil -} - -func (g *grantDb) Insert(in *dbmodel.Grant) error { - err := g.db.Create(in).Error - if err != nil { - log.Error("fail to insert the grant", zap.Any("in", in), zap.Error(err)) - } - return err -} - -func (g *grantDb) Delete(tenantID string, roleID int64, object string, objectName string) error { - err := g.db.Model(dbmodel.Grant{}). - Where(&dbmodel.Grant{RoleID: roleID, Object: object, ObjectName: objectName}). - Where(dbmodel.GetCommonCondition(tenantID, false)). - Update("is_deleted", true).Error - if err != nil { - log.Error("fail to delete the grant", zap.String("tenant_id", tenantID), zap.Int64("roleID", roleID), zap.String("object", object), zap.String("object_name", objectName), zap.Error(err)) - } - return err -} diff --git a/internal/metastore/db/dao/grant_test.go b/internal/metastore/db/dao/grant_test.go deleted file mode 100644 index 8ca2d1d069..0000000000 --- a/internal/metastore/db/dao/grant_test.go +++ /dev/null @@ -1,203 +0,0 @@ -package dao - -import ( - "testing" - - "github.com/cockroachdb/errors" - - "github.com/DATA-DOG/go-sqlmock" - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - "github.com/stretchr/testify/assert" -) - -func TestGrant_GetGrants(t *testing.T) { - var ( - roleID1 = 10 - roleID2 = 20 - object = "Collection" - objectName = "col1" - grants []*dbmodel.Grant - err error - ) - - mock.ExpectQuery("SELECT * FROM `grant` WHERE `is_deleted` = ? AND `tenant_id` = ?"). - WithArgs(false, tenantID). - WillReturnRows( - sqlmock.NewRows([]string{"tenant_id", "role_id", "object", "object_name"}). - AddRow(tenantID, roleID1, object, objectName). - AddRow(tenantID, roleID2, object, objectName)) - - mock.ExpectQuery("SELECT * FROM `role` WHERE `role`.`id` IN (?,?)"). - WithArgs(roleID1, roleID2). - WillReturnRows( - sqlmock.NewRows([]string{"id", "tenant_id", "name"}). - AddRow(roleID1, tenantID, "foo1"). - AddRow(roleID2, tenantID, "foo2")) - - grants, err = grantTestDb.GetGrants(tenantID, 0, "", "") - mock.MatchExpectationsInOrder(false) - assert.NoError(t, err) - assert.Equal(t, 2, len(grants)) - assert.Equal(t, "foo2", grants[1].Role.Name) - assert.Equal(t, object, grants[0].Object) - assert.Equal(t, objectName, grants[0].ObjectName) - - mock.ExpectQuery("SELECT * FROM `grant` WHERE `is_deleted` = ? AND `tenant_id` = ?"). - WithArgs(false, tenantID). - WillReturnError(errors.New("test error")) - _, err = grantTestDb.GetGrants(tenantID, 0, "", "") - mock.MatchExpectationsInOrder(false) - assert.Error(t, err) -} - -func TestGrant_GetGrantsWithRoleID(t *testing.T) { - var ( - roleID1 = 10 - object1 = "Collection" - objectName1 = "col1" - object2 = "Global" - objectName2 = "*" - grants []*dbmodel.Grant - err error - ) - - mock.ExpectQuery("SELECT * FROM `grant` WHERE `grant`.`role_id` = ? AND `is_deleted` = ? AND `tenant_id` = ?"). - WithArgs(roleID1, false, tenantID). - WillReturnRows( - sqlmock.NewRows([]string{"tenant_id", "role_id", "object", "object_name"}). - AddRow(tenantID, roleID1, object1, objectName1). - AddRow(tenantID, roleID1, object2, objectName2)) - - mock.ExpectQuery("SELECT * FROM `role` WHERE `role`.`id` = ?"). - WithArgs(roleID1). - WillReturnRows( - sqlmock.NewRows([]string{"id", "tenant_id", "name"}). - AddRow(roleID1, tenantID, "foo1")) - - grants, err = grantTestDb.GetGrants(tenantID, int64(roleID1), "", "") - mock.MatchExpectationsInOrder(false) - assert.NoError(t, err) - assert.Equal(t, 2, len(grants)) - assert.Equal(t, "foo1", grants[0].Role.Name) - assert.Equal(t, object1, grants[0].Object) - assert.Equal(t, objectName2, grants[1].ObjectName) -} - -func TestGrant_GetGrantsWithObject(t *testing.T) { - var ( - roleID = 10 - object = "Collection" - objectName = "col1" - grants []*dbmodel.Grant - err error - ) - - mock.ExpectQuery("SELECT * FROM `grant` WHERE `grant`.`role_id` = ? AND `grant`.`object` = ? AND `grant`.`object_name` = ? AND `is_deleted` = ? AND `tenant_id` = ?"). - WithArgs(roleID, object, objectName, false, tenantID). - WillReturnRows( - sqlmock.NewRows([]string{"tenant_id", "role_id", "object", "object_name"}). - AddRow(tenantID, roleID, object, objectName)) - - mock.ExpectQuery("SELECT * FROM `role` WHERE `role`.`id` = ?"). - WithArgs(roleID). - WillReturnRows( - sqlmock.NewRows([]string{"id", "tenant_id", "name"}). - AddRow(roleID, tenantID, "foo1")) - - grants, err = grantTestDb.GetGrants(tenantID, int64(roleID), object, objectName) - mock.MatchExpectationsInOrder(false) - assert.NoError(t, err) - assert.Equal(t, 1, len(grants)) - assert.Equal(t, "foo1", grants[0].Role.Name) - assert.Equal(t, object, grants[0].Object) - assert.Equal(t, objectName, grants[0].ObjectName) -} - -func TestGrant_Insert(t *testing.T) { - var ( - grant *dbmodel.Grant - err error - ) - grant = &dbmodel.Grant{ - Base: GetBase(), - RoleID: 1, - Object: "Global", - ObjectName: "Col", - } - - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `grant` (`tenant_id`,`is_deleted`,`created_at`,`updated_at`,`role_id`,`object`,`object_name`) VALUES (?,?,?,?,?,?,?)"). - WithArgs(grant.TenantID, grant.IsDeleted, grant.CreatedAt, grant.UpdatedAt, grant.RoleID, grant.Object, grant.ObjectName). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - err = grantTestDb.Insert(grant) - mock.MatchExpectationsInOrder(false) - assert.NoError(t, err) - -} - -func TestGrant_Insert_Error(t *testing.T) { - var ( - grant *dbmodel.Grant - err error - ) - grant = &dbmodel.Grant{ - Base: GetBase(), - RoleID: 1, - Object: "Global", - ObjectName: "Col", - } - - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `grant` (`tenant_id`,`is_deleted`,`created_at`,`updated_at`,`role_id`,`object`,`object_name`) VALUES (?,?,?,?,?,?,?)"). - WithArgs(grant.TenantID, grant.IsDeleted, grant.CreatedAt, grant.UpdatedAt, grant.RoleID, grant.Object, grant.ObjectName). - WillReturnError(errors.New("test error")) - mock.ExpectRollback() - err = grantTestDb.Insert(grant) - mock.MatchExpectationsInOrder(false) - assert.Error(t, err) -} - -func TestGrant_Delete(t *testing.T) { - var ( - grant *dbmodel.Grant - err error - ) - grant = &dbmodel.Grant{ - Base: GetBase(), - RoleID: 1, - Object: "Global", - ObjectName: "Col", - } - - mock.ExpectBegin() - mock.ExpectExec("UPDATE `grant` SET `is_deleted`=?,`updated_at`=? WHERE `grant`.`role_id` = ? AND `grant`.`object` = ? AND `grant`.`object_name` = ? AND `is_deleted` = ? AND `tenant_id` = ?"). - WithArgs(true, AnyTime{}, grant.RoleID, grant.Object, grant.ObjectName, false, grant.TenantID). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - err = grantTestDb.Delete(grant.TenantID, grant.RoleID, grant.Object, grant.ObjectName) - mock.MatchExpectationsInOrder(false) - assert.NoError(t, err) -} - -func TestGrant_Delete_Error(t *testing.T) { - var ( - grant *dbmodel.Grant - err error - ) - grant = &dbmodel.Grant{ - Base: GetBase(), - RoleID: 1, - Object: "Global", - ObjectName: "Col", - } - - mock.ExpectBegin() - mock.ExpectExec("UPDATE `grant` SET `is_deleted`=?,`updated_at`=? WHERE `grant`.`role_id` = ? AND `grant`.`object` = ? AND `grant`.`object_name` = ? AND `is_deleted` = ? AND `tenant_id` = ?"). - WithArgs(true, AnyTime{}, grant.RoleID, grant.Object, grant.ObjectName, false, grant.TenantID). - WillReturnError(errors.New("test error")) - mock.ExpectCommit() - err = grantTestDb.Delete(grant.TenantID, grant.RoleID, grant.Object, grant.ObjectName) - mock.MatchExpectationsInOrder(false) - assert.Error(t, err) -} diff --git a/internal/metastore/db/dao/index.go b/internal/metastore/db/dao/index.go deleted file mode 100644 index c653e50cb9..0000000000 --- a/internal/metastore/db/dao/index.go +++ /dev/null @@ -1,93 +0,0 @@ -package dao - -import ( - "go.uber.org/zap" - "gorm.io/gorm" - - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - "github.com/milvus-io/milvus/pkg/log" - "github.com/milvus-io/milvus/pkg/util/typeutil" -) - -type indexDb struct { - db *gorm.DB -} - -func (s *indexDb) Get(tenantID string, collectionID typeutil.UniqueID) ([]*dbmodel.Index, error) { - var r []*dbmodel.Index - - err := s.db.Model(&dbmodel.Index{}).Where("tenant_id = ? AND collection_id = ?", tenantID, collectionID).Find(&r).Error - if err != nil { - log.Error("get indexes by collection_id failed", zap.String("tenant", tenantID), zap.Int64("collectionID", collectionID), zap.Error(err)) - return nil, err - } - - return r, nil -} - -func (s *indexDb) List(tenantID string) ([]*dbmodel.IndexResult, error) { - tx := s.db.Table("indexes"). - Select("indexes.field_id AS field_id, indexes.collection_id AS collection_id, indexes.index_id AS index_id, "+ - "indexes.index_name AS index_name, indexes.index_params AS index_params, indexes.type_params AS type_params, "+ - "indexes.is_deleted AS is_deleted, indexes.create_time AS create_time"). - Where("indexes.tenant_id = ?", tenantID) - - var rs []*dbmodel.IndexResult - err := tx.Scan(&rs).Error - if err != nil { - log.Error("list indexes by join failed", zap.String("tenant", tenantID), zap.Error(err)) - return nil, err - } - - return rs, nil -} - -func (s *indexDb) Insert(in []*dbmodel.Index) error { - err := s.db.CreateInBatches(in, 100).Error - if err != nil { - log.Error("insert index failed", zap.Error(err)) - return err - } - - return nil -} - -func (s *indexDb) Update(in *dbmodel.Index) error { - err := s.db.Model(&dbmodel.Index{}).Where("tenant_id = ? AND collection_id = ? AND index_id = ?", in.TenantID, in.CollectionID, in.IndexID).Updates(dbmodel.Index{ - CreateTime: in.CreateTime, // if in.CreateTime is not set, column CreateTime will not be updated - IsDeleted: in.IsDeleted, - }).Error - - if err != nil { - log.Error("update indexes failed", zap.String("tenant", in.TenantID), zap.Int64("collectionID", in.CollectionID), zap.Int64("indexID", in.IndexID), zap.Error(err)) - return err - } - - return nil -} - -func (s *indexDb) MarkDeletedByCollectionID(tenantID string, collID typeutil.UniqueID) error { - err := s.db.Model(&dbmodel.Index{}).Where("tenant_id = ? AND collection_id = ?", tenantID, collID).Updates(dbmodel.Index{ - IsDeleted: true, - }).Error - - if err != nil { - log.Error("update indexes is_deleted=true failed", zap.String("tenant", tenantID), zap.Int64("collectionID", collID), zap.Error(err)) - return err - } - - return nil -} - -func (s *indexDb) MarkDeletedByIndexID(tenantID string, indexID typeutil.UniqueID) error { - err := s.db.Model(&dbmodel.Index{}).Where("tenant_id = ? AND index_id = ?", tenantID, indexID).Updates(dbmodel.Index{ - IsDeleted: true, - }).Error - - if err != nil { - log.Error("update indexes is_deleted=true failed", zap.String("tenant", tenantID), zap.Int64("indexID", indexID), zap.Error(err)) - return err - } - - return nil -} diff --git a/internal/metastore/db/dao/index_test.go b/internal/metastore/db/dao/index_test.go deleted file mode 100644 index b055d66f17..0000000000 --- a/internal/metastore/db/dao/index_test.go +++ /dev/null @@ -1,248 +0,0 @@ -package dao - -import ( - "testing" - "time" - - "github.com/cockroachdb/errors" - - "github.com/DATA-DOG/go-sqlmock" - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - "github.com/stretchr/testify/assert" -) - -func TestIndex_Get(t *testing.T) { - var indexes = []*dbmodel.Index{ - { - TenantID: "", - FieldID: fieldID1, - CollectionID: collID1, - IndexID: indexID1, - IndexName: "test_index_1", - IndexParams: "", - TypeParams: "", - UserIndexParams: "", - IsAutoIndex: false, - }, - } - - // expectation - mock.ExpectQuery("SELECT * FROM `indexes` WHERE tenant_id = ? AND collection_id = ?"). - WithArgs(tenantID, collID1). - WillReturnRows( - sqlmock.NewRows([]string{"tenant_id", "field_id", "collection_id", "index_id", "index_name", "index_params", "type_params"}). - AddRow(indexes[0].TenantID, indexes[0].FieldID, indexes[0].CollectionID, indexes[0].IndexID, indexes[0].IndexName, indexes[0].IndexParams, indexes[0].TypeParams)) - - // actual - res, err := indexTestDb.Get(tenantID, collID1) - assert.NoError(t, err) - assert.Equal(t, indexes, res) -} - -func TestIndex_Get_Error(t *testing.T) { - // expectation - mock.ExpectQuery("SELECT * FROM `indexes` WHERE tenant_id = ? AND collection_id = ?"). - WithArgs(tenantID, collID1). - WillReturnError(errors.New("test error")) - - // actual - res, err := indexTestDb.Get(tenantID, collID1) - assert.Nil(t, res) - assert.Error(t, err) -} - -func TestIndex_List(t *testing.T) { - var indexResults = []*dbmodel.IndexResult{ - { - FieldID: fieldID1, - CollectionID: collID1, - IndexID: indexID1, - IndexName: "test_index_1", - TypeParams: "", - IndexParams: "", - UserIndexParams: "", - IsAutoIndex: false, - CreateTime: uint64(1011), - IsDeleted: false, - }, - } - - // expectation - mock.ExpectQuery("SELECT indexes.field_id AS field_id, indexes.collection_id AS collection_id, indexes.index_id AS index_id, indexes.index_name AS index_name, indexes.index_params AS index_params, indexes.type_params AS type_params, indexes.is_deleted AS is_deleted, indexes.create_time AS create_time FROM `indexes` WHERE indexes.tenant_id = ?"). - WithArgs(tenantID). - WillReturnRows( - sqlmock.NewRows([]string{"field_id", "collection_id", "index_id", "index_name", "index_params", "type_params", "user_index_params", "is_auto_index", "is_deleted", "create_time"}). - AddRow(indexResults[0].FieldID, indexResults[0].CollectionID, indexResults[0].IndexID, indexResults[0].IndexName, indexResults[0].IndexParams, indexResults[0].TypeParams, indexResults[0].UserIndexParams, indexResults[0].IsAutoIndex, indexResults[0].IsDeleted, indexResults[0].CreateTime)) - - // actual - res, err := indexTestDb.List(tenantID) - assert.NoError(t, err) - assert.Equal(t, indexResults, res) -} - -func TestIndex_List_Error(t *testing.T) { - // expectation - mock.ExpectQuery("SELECT indexes.field_id AS field_id, indexes.collection_id AS collection_id, indexes.index_id AS index_id, indexes.index_name AS index_name, indexes.index_params AS index_params, indexes.type_params AS type_params, indexes.is_deleted AS is_deleted, indexes.create_time AS create_time FROM `indexes` WHERE indexes.tenant_id = ?"). - WithArgs(tenantID). - WillReturnError(errors.New("test error")) - - // actual - res, err := indexTestDb.List(tenantID) - assert.Nil(t, res) - assert.Error(t, err) -} - -func TestIndex_Insert(t *testing.T) { - var indexes = []*dbmodel.Index{ - { - TenantID: tenantID, - FieldID: fieldID1, - CollectionID: collID1, - IndexID: indexID1, - IndexName: "test_index_1", - IndexParams: "", - TypeParams: "", - UserIndexParams: "", - IsAutoIndex: false, - CreateTime: uint64(1011), - IsDeleted: false, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - }, - } - - // expectation - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `indexes` (`tenant_id`,`field_id`,`collection_id`,`index_id`,`index_name`,`index_params`,`type_params`,`user_index_params`,`is_auto_index`,`create_time`,`is_deleted`,`created_at`,`updated_at`) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)"). - WithArgs(indexes[0].TenantID, indexes[0].FieldID, indexes[0].CollectionID, indexes[0].IndexID, indexes[0].IndexName, indexes[0].IndexParams, indexes[0].TypeParams, indexes[0].UserIndexParams, indexes[0].IsAutoIndex, indexes[0].CreateTime, indexes[0].IsDeleted, indexes[0].CreatedAt, indexes[0].UpdatedAt). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - - // actual - err := indexTestDb.Insert(indexes) - assert.NoError(t, err) -} - -func TestIndex_Insert_Error(t *testing.T) { - var indexes = []*dbmodel.Index{ - { - TenantID: tenantID, - FieldID: fieldID1, - CollectionID: collID1, - IndexID: indexID1, - IndexName: "test_index_1", - IndexParams: "", - CreateTime: uint64(1011), - IsDeleted: false, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - }, - } - - // expectation - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `indexes` (`tenant_id`,`field_id`,`collection_id`,`index_id`,`index_name`,`index_params`,`type_params`, `user_index_params`, `is_auto_index`, `create_time`,`is_deleted`,`created_at`,`updated_at`) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?)"). - WithArgs(indexes[0].TenantID, indexes[0].FieldID, indexes[0].CollectionID, indexes[0].IndexID, indexes[0].IndexName, indexes[0].IndexParams, indexes[0].TypeParams, indexes[0].UserIndexParams, indexes[0].IsAutoIndex, indexes[0].CreateTime, indexes[0].IsDeleted, indexes[0].CreatedAt, indexes[0].UpdatedAt). - WillReturnError(errors.New("test error")) - mock.ExpectRollback() - - // actual - err := indexTestDb.Insert(indexes) - assert.Error(t, err) -} - -func TestIndex_Update(t *testing.T) { - var index = &dbmodel.Index{ - TenantID: tenantID, - IndexName: "test_index_name_1", - IndexID: indexID1, - IndexParams: "", - IsDeleted: true, - CreateTime: uint64(1112), - } - - // expectation - mock.ExpectBegin() - mock.ExpectExec("UPDATE `indexes` SET `create_time`=?,`is_deleted`=?,`updated_at`=? WHERE tenant_id = ? AND collection_id = ? AND index_id = ?"). - WithArgs(index.CreateTime, index.IsDeleted, AnyTime{}, index.TenantID, index.CollectionID, index.IndexID). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - - // actual - err := indexTestDb.Update(index) - assert.NoError(t, err) -} - -func TestIndex_Update_Error(t *testing.T) { - var index = &dbmodel.Index{ - TenantID: tenantID, - IndexName: "test_index_name_1", - IndexID: indexID1, - IndexParams: "", - IsDeleted: false, - CreateTime: uint64(1112), - } - - // expectation - mock.ExpectBegin() - mock.ExpectExec("UPDATE `indexes` SET `create_time`=?,`updated_at`=? WHERE tenant_id = ? AND collection_id = ? AND index_id = ?"). - WithArgs(index.CreateTime, AnyTime{}, index.TenantID, index.CollectionID, index.IndexID). - WillReturnError(errors.New("test error")) - mock.ExpectRollback() - - // actual - err := indexTestDb.Update(index) - assert.Error(t, err) -} - -func TestIndex_MarkDeletedByCollID(t *testing.T) { - // expectation - mock.ExpectBegin() - mock.ExpectExec("UPDATE `indexes` SET `is_deleted`=?,`updated_at`=? WHERE tenant_id = ? AND collection_id = ?"). - WithArgs(true, AnyTime{}, tenantID, collID1). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - - // actual - err := indexTestDb.MarkDeletedByCollectionID(tenantID, collID1) - assert.NoError(t, err) -} - -func TestIndex_MarkDeletedByCollID_Error(t *testing.T) { - // expectation - mock.ExpectBegin() - mock.ExpectExec("UPDATE `indexes` SET `is_deleted`=?,`updated_at`=? WHERE tenant_id = ? AND collection_id = ?"). - WithArgs(true, AnyTime{}, tenantID, collID1). - WillReturnError(errors.New("test error")) - mock.ExpectRollback() - - // actual - err := indexTestDb.MarkDeletedByCollectionID(tenantID, collID1) - assert.Error(t, err) -} - -func TestIndex_MarkDeletedByIdxID(t *testing.T) { - // expectation - mock.ExpectBegin() - mock.ExpectExec("UPDATE `indexes` SET `is_deleted`=?,`updated_at`=? WHERE tenant_id = ? AND index_id = ?"). - WithArgs(true, AnyTime{}, tenantID, indexID1). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - - // actual - err := indexTestDb.MarkDeletedByIndexID(tenantID, indexID1) - assert.NoError(t, err) -} - -func TestIndex_MarkDeletedByIdxID_Error(t *testing.T) { - // expectation - mock.ExpectBegin() - mock.ExpectExec("UPDATE `indexes` SET `is_deleted`=?,`updated_at`=? WHERE tenant_id = ? AND index_id = ?"). - WithArgs(true, AnyTime{}, tenantID, indexID1). - WillReturnError(errors.New("test error")) - mock.ExpectRollback() - - // actual - err := indexTestDb.MarkDeletedByIndexID(tenantID, indexID1) - assert.Error(t, err) -} diff --git a/internal/metastore/db/dao/partition.go b/internal/metastore/db/dao/partition.go deleted file mode 100644 index c32424240f..0000000000 --- a/internal/metastore/db/dao/partition.go +++ /dev/null @@ -1,57 +0,0 @@ -package dao - -import ( - "go.uber.org/zap" - "gorm.io/gorm" - - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - "github.com/milvus-io/milvus/pkg/log" - "github.com/milvus-io/milvus/pkg/util/typeutil" -) - -type partitionDb struct { - db *gorm.DB -} - -func (s *partitionDb) GetByCollectionID(tenantID string, collectionID typeutil.UniqueID, ts typeutil.Timestamp) ([]*dbmodel.Partition, error) { - var r []*dbmodel.Partition - - err := s.db.Model(&dbmodel.Partition{}).Where("tenant_id = ? AND collection_id = ? AND ts = ? AND is_deleted = false", tenantID, collectionID, ts).Find(&r).Error - if err != nil { - log.Error("get partitions by collection_id and ts failed", zap.String("tenant", tenantID), zap.Int64("collectionID", collectionID), zap.Uint64("ts", ts), zap.Error(err)) - return nil, err - } - - return r, nil -} - -func (s *partitionDb) Insert(in []*dbmodel.Partition) error { - err := s.db.CreateInBatches(in, 100).Error - if err != nil { - log.Error("insert partition failed", zap.Error(err)) - return err - } - - return nil -} - -func generatePartitionUpdatesWithoutID(in *dbmodel.Partition) map[string]interface{} { - ret := map[string]interface{}{ - "tenant_id": in.TenantID, - "partition_id": in.PartitionID, - "partition_name": in.PartitionName, - "partition_created_timestamp": in.PartitionCreatedTimestamp, - "collection_id": in.CollectionID, - "status": in.Status, - "ts": in.Ts, - "is_deleted": in.IsDeleted, - "created_at": in.CreatedAt, - "updated_at": in.UpdatedAt, - } - return ret -} - -func (s *partitionDb) Update(in *dbmodel.Partition) error { - updates := generatePartitionUpdatesWithoutID(in) - return s.db.Model(&dbmodel.Partition{}).Where("id = ?", in.ID).Updates(updates).Error -} diff --git a/internal/metastore/db/dao/partition_test.go b/internal/metastore/db/dao/partition_test.go deleted file mode 100644 index 84374beedf..0000000000 --- a/internal/metastore/db/dao/partition_test.go +++ /dev/null @@ -1,155 +0,0 @@ -package dao - -import ( - "testing" - "time" - - "github.com/cockroachdb/errors" - - "github.com/DATA-DOG/go-sqlmock" - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - "github.com/milvus-io/milvus/pkg/util/typeutil" - "github.com/stretchr/testify/assert" -) - -func TestPartition_GetByCollID(t *testing.T) { - var partitions = []*dbmodel.Partition{ - { - TenantID: tenantID, - PartitionID: fieldID1, - PartitionName: "test_field_1", - PartitionCreatedTimestamp: typeutil.Timestamp(1000), - CollectionID: collID1, - Ts: ts, - }, - } - - // expectation - mock.ExpectQuery("SELECT * FROM `partitions` WHERE tenant_id = ? AND collection_id = ? AND ts = ? AND is_deleted = false"). - WithArgs(tenantID, collID1, ts). - WillReturnRows( - sqlmock.NewRows([]string{"tenant_id", "partition_id", "partition_name", "partition_created_timestamp", "collection_id", "ts"}). - AddRow(partitions[0].TenantID, partitions[0].PartitionID, partitions[0].PartitionName, partitions[0].PartitionCreatedTimestamp, partitions[0].CollectionID, partitions[0].Ts)) - - // actual - res, err := partitionTestDb.GetByCollectionID(tenantID, collID1, ts) - assert.NoError(t, err) - assert.Equal(t, partitions, res) -} - -func TestPartition_GetByCollID_Error(t *testing.T) { - // expectation - mock.ExpectQuery("SELECT * FROM `partitions` WHERE tenant_id = ? AND collection_id = ? AND ts = ? AND is_deleted = false"). - WithArgs(tenantID, collID1, ts). - WillReturnError(errors.New("test error")) - - // actual - res, err := partitionTestDb.GetByCollectionID(tenantID, collID1, ts) - assert.Nil(t, res) - assert.Error(t, err) -} - -func TestPartition_Insert(t *testing.T) { - var partitions = []*dbmodel.Partition{ - { - TenantID: tenantID, - PartitionID: fieldID1, - PartitionName: "test_field_1", - PartitionCreatedTimestamp: typeutil.Timestamp(1000), - CollectionID: collID1, - Ts: ts, - IsDeleted: false, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - }, - } - - // expectation - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `partitions` (`tenant_id`,`partition_id`,`partition_name`,`partition_created_timestamp`,`collection_id`,`status`,`ts`,`is_deleted`,`created_at`,`updated_at`) VALUES (?,?,?,?,?,?,?,?,?,?)"). - WithArgs(partitions[0].TenantID, partitions[0].PartitionID, partitions[0].PartitionName, partitions[0].PartitionCreatedTimestamp, partitions[0].CollectionID, partitions[0].Status, partitions[0].Ts, partitions[0].IsDeleted, partitions[0].CreatedAt, partitions[0].UpdatedAt). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - - // actual - err := partitionTestDb.Insert(partitions) - assert.NoError(t, err) -} - -func TestPartition_Insert_Error(t *testing.T) { - var partitions = []*dbmodel.Partition{ - { - TenantID: tenantID, - PartitionID: fieldID1, - PartitionName: "test_field_1", - PartitionCreatedTimestamp: typeutil.Timestamp(1000), - CollectionID: collID1, - Ts: ts, - IsDeleted: false, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - }, - } - - // expectation - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `partitions` (`tenant_id`,`partition_id`,`partition_name`,`partition_created_timestamp`,`collection_id`,`ts`,`is_deleted`,`created_at`,`updated_at`) VALUES (?,?,?,?,?,?,?,?,?)"). - WithArgs(partitions[0].TenantID, partitions[0].PartitionID, partitions[0].PartitionName, partitions[0].PartitionCreatedTimestamp, partitions[0].CollectionID, partitions[0].Ts, partitions[0].IsDeleted, partitions[0].CreatedAt, partitions[0].UpdatedAt). - WillReturnError(errors.New("test error")) - mock.ExpectRollback() - - // actual - err := partitionTestDb.Insert(partitions) - assert.Error(t, err) -} - -func Test_partitionDb_Update(t *testing.T) { - t.Run("normal case", func(t *testing.T) { - partition := &dbmodel.Partition{ - ID: 100, - TenantID: tenantID, - PartitionID: fieldID1, - PartitionName: "test_field_1", - PartitionCreatedTimestamp: typeutil.Timestamp(1000), - CollectionID: collID1, - Ts: ts, - IsDeleted: false, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - } - - mock.ExpectBegin() - mock.ExpectExec("UPDATE `partitions` SET `collection_id`=?,`created_at`=?,`is_deleted`=?,`partition_created_timestamp`=?,`partition_id`=?,`partition_name`=?,`status`=?,`tenant_id`=?,`ts`=?,`updated_at`=? WHERE id = ?"). - WithArgs(partition.CollectionID, partition.CreatedAt, partition.IsDeleted, partition.PartitionCreatedTimestamp, partition.PartitionID, partition.PartitionName, partition.Status, partition.TenantID, partition.Ts, partition.UpdatedAt, partition.ID). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - - err := partitionTestDb.Update(partition) - assert.NoError(t, err) - }) - - t.Run("error case", func(t *testing.T) { - partition := &dbmodel.Partition{ - ID: 100, - TenantID: tenantID, - PartitionID: fieldID1, - PartitionName: "test_field_1", - PartitionCreatedTimestamp: typeutil.Timestamp(1000), - CollectionID: collID1, - Ts: ts, - IsDeleted: false, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - } - - mock.ExpectBegin() - mock.ExpectExec("UPDATE `partitions` SET `collection_id`=?,`created_at`=?,`is_deleted`=?,`partition_created_timestamp`=?,`partition_id`=?,`partition_name`=?,`status`=?,`tenant_id`=?,`ts`=?,`updated_at`=? WHERE id = ?"). - WithArgs(partition.CollectionID, partition.CreatedAt, partition.IsDeleted, partition.PartitionCreatedTimestamp, partition.PartitionID, partition.PartitionName, partition.Status, partition.TenantID, partition.Ts, partition.UpdatedAt, partition.ID). - WillReturnError(errors.New("error mock Update Partition")) - mock.ExpectRollback() - - err := partitionTestDb.Update(partition) - assert.Error(t, err) - }) - -} diff --git a/internal/metastore/db/dao/role.go b/internal/metastore/db/dao/role.go deleted file mode 100644 index 564f2911f9..0000000000 --- a/internal/metastore/db/dao/role.go +++ /dev/null @@ -1,48 +0,0 @@ -package dao - -import ( - "go.uber.org/zap" - "gorm.io/gorm" - - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - "github.com/milvus-io/milvus/pkg/log" -) - -type roleDb struct { - db *gorm.DB -} - -func (r *roleDb) GetRoles(tenantID string, name string) ([]*dbmodel.Role, error) { - var ( - roles []*dbmodel.Role - err error - ) - err = r.db.Model(&dbmodel.Role{}). - Where(&dbmodel.Role{Name: name}). - Where(dbmodel.GetCommonCondition(tenantID, false)). - Find(&roles).Error - if err != nil { - log.Error("fail to get roles", zap.String("tenant_id", tenantID), zap.String("name", name), zap.Error(err)) - return nil, err - } - return roles, nil -} - -func (r *roleDb) Insert(in *dbmodel.Role) error { - err := r.db.Create(in).Error - if err != nil { - log.Error("fail to insert the role", zap.Any("in", in), zap.Error(err)) - } - return err -} - -func (r *roleDb) Delete(tenantID string, name string) error { - err := r.db.Model(dbmodel.Role{}). - Where(&dbmodel.Role{Name: name}). - Where(dbmodel.GetCommonCondition(tenantID, false)). - Update("is_deleted", true).Error - if err != nil { - log.Error("fail to delete the role", zap.String("tenant_id", tenantID), zap.String("name", name), zap.Error(err)) - } - return err -} diff --git a/internal/metastore/db/dao/role_test.go b/internal/metastore/db/dao/role_test.go deleted file mode 100644 index 5b2fc9c770..0000000000 --- a/internal/metastore/db/dao/role_test.go +++ /dev/null @@ -1,133 +0,0 @@ -package dao - -import ( - "testing" - - "github.com/cockroachdb/errors" - - "github.com/stretchr/testify/assert" - - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - - "github.com/DATA-DOG/go-sqlmock" -) - -func TestRole_GetRoles(t *testing.T) { - var ( - roles []*dbmodel.Role - err error - ) - - mock.ExpectQuery("SELECT * FROM `role` WHERE `is_deleted` = ? AND `tenant_id` = ?"). - WithArgs(false, tenantID). - WillReturnRows( - sqlmock.NewRows([]string{"tenant_id", "name"}). - AddRow(tenantID, "foo1"). - AddRow(tenantID, "foo2")) - - roles, err = roleTestDb.GetRoles(tenantID, "") - assert.NoError(t, err) - assert.Equal(t, 2, len(roles)) -} - -func TestRole_GetRoles_Error(t *testing.T) { - mock.ExpectQuery("SELECT * FROM `role` WHERE `is_deleted` = ? AND `tenant_id` = ?"). - WithArgs(false, tenantID). - WillReturnError(errors.New("test error")) - _, err := roleTestDb.GetRoles(tenantID, "") - assert.Error(t, err) -} - -func TestRole_GetRoles_WithRoleName(t *testing.T) { - var ( - roleName = "foo1" - roles []*dbmodel.Role - err error - ) - mock.ExpectQuery("SELECT * FROM `role` WHERE `role`.`name` = ? AND `is_deleted` = ? AND `tenant_id` = ?"). - WithArgs(roleName, false, tenantID). - WillReturnRows( - sqlmock.NewRows([]string{"tenant_id", "name"}). - AddRow(tenantID, roleName)) - roles, err = roleTestDb.GetRoles(tenantID, roleName) - assert.NoError(t, err) - assert.Equal(t, 1, len(roles)) - assert.Equal(t, roleName, roles[0].Name) -} - -func TestRole_Insert(t *testing.T) { - var ( - role *dbmodel.Role - err error - ) - role = &dbmodel.Role{ - Base: GetBase(), - Name: "foo", - } - - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `role` (`tenant_id`,`is_deleted`,`created_at`,`updated_at`,`name`) VALUES (?,?,?,?,?)"). - WithArgs(role.TenantID, role.IsDeleted, role.CreatedAt, role.UpdatedAt, role.Name). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - - err = roleTestDb.Insert(role) - assert.NoError(t, err) - -} - -func TestRole_Insert_Error(t *testing.T) { - var ( - role *dbmodel.Role - err error - ) - role = &dbmodel.Role{ - Base: GetBase(), - Name: "foo", - } - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `role` (`tenant_id`,`is_deleted`,`created_at`,`updated_at`,`name`) VALUES (?,?,?,?,?)"). - WithArgs(role.TenantID, role.IsDeleted, role.CreatedAt, role.UpdatedAt, role.Name). - WillReturnError(errors.New("test error")) - mock.ExpectRollback() - err = roleTestDb.Insert(role) - assert.Error(t, err) -} - -func TestRole_Delete(t *testing.T) { - var ( - role *dbmodel.Role - err error - ) - role = &dbmodel.Role{ - Base: GetBase(), - Name: "foo", - } - mock.ExpectBegin() - mock.ExpectExec("UPDATE `role` SET `is_deleted`=?,`updated_at`=? WHERE `role`.`name` = ? AND `is_deleted` = ? AND `tenant_id` = ?"). - WithArgs(true, AnyTime{}, role.Name, role.IsDeleted, role.TenantID). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - - err = roleTestDb.Delete(role.TenantID, role.Name) - assert.NoError(t, err) -} - -func TestRole_Delete_Error(t *testing.T) { - var ( - role *dbmodel.Role - err error - ) - role = &dbmodel.Role{ - Base: GetBase(), - Name: "foo", - } - mock.ExpectBegin() - mock.ExpectExec("UPDATE `role` SET `is_deleted`=?,`updated_at`=? WHERE `role`.`name` = ? AND `is_deleted` = ? AND `tenant_id` = ?"). - WithArgs(true, AnyTime{}, role.Name, role.IsDeleted, role.TenantID). - WillReturnError(errors.New("test error")) - mock.ExpectRollback() - - err = roleTestDb.Delete(role.TenantID, role.Name) - assert.Error(t, err) -} diff --git a/internal/metastore/db/dao/segment_index.go b/internal/metastore/db/dao/segment_index.go deleted file mode 100644 index e45b0d11ac..0000000000 --- a/internal/metastore/db/dao/segment_index.go +++ /dev/null @@ -1,125 +0,0 @@ -package dao - -import ( - "go.uber.org/zap" - "gorm.io/gorm" - "gorm.io/gorm/clause" - - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - "github.com/milvus-io/milvus/pkg/log" - "github.com/milvus-io/milvus/pkg/util/typeutil" -) - -type segmentIndexDb struct { - db *gorm.DB -} - -func (s *segmentIndexDb) Get(tenantID string, collectionID, buildID typeutil.UniqueID) ([]*dbmodel.SegmentIndexResult, error) { - var r []*dbmodel.SegmentIndexResult - - err := s.db.Model(&dbmodel.SegmentIndex{}).Where("tenant_id = ? AND collection_id = ? AND build_id = ?", tenantID, collectionID, buildID).Find(&r).Error - if err != nil { - log.Error("get indexes by collection_id failed", zap.String("tenant", tenantID), zap.Int64("collectionID", collectionID), zap.Error(err)) - return nil, err - } - - return r, nil -} - -func (s *segmentIndexDb) List(tenantID string) ([]*dbmodel.SegmentIndexResult, error) { - tx := s.db.Table("segment_indexes"). - Select("segment_indexes.collection_id AS collection_id, segment_indexes.partition_id AS partition_id, "+ - "segment_indexes.segment_id AS segment_id, segment_indexes.num_rows AS num_rows, segment_indexes.index_id AS index_id, "+ - "segment_indexes.build_id AS build_id, segment_indexes.node_id AS node_id, segment_indexes.index_version AS index_version, "+ - "segment_indexes.index_state AS index_state,segment_indexes.fail_reason AS fail_reason, segment_indexes.create_time AS create_time,"+ - "segment_indexes.index_file_keys AS index_file_keys, segment_indexes.index_size AS index_size, segment_indexes.is_deleted AS is_deleted"). - Where("indexes.tenant_id = ?", tenantID) - - var rs []*dbmodel.SegmentIndexResult - err := tx.Scan(&rs).Error - if err != nil { - log.Error("list indexes by join failed", zap.String("tenant", tenantID), zap.Error(err)) - return nil, err - } - - return rs, nil -} - -func (s *segmentIndexDb) Insert(in []*dbmodel.SegmentIndex) error { - err := s.db.CreateInBatches(in, 100).Error - if err != nil { - log.Error("insert segment_indexes failed", zap.Error(err)) - return err - } - - return nil -} - -func (s *segmentIndexDb) Update(in *dbmodel.SegmentIndex) error { - err := s.db.CreateInBatches(in, 100).Error - if err != nil { - log.Error("insert segment_indexes failed", zap.Error(err)) - return err - } - - return nil -} - -func (s *segmentIndexDb) Upsert(in []*dbmodel.SegmentIndex) error { - err := s.db.Clauses(clause.OnConflict{ - // constraint UNIQUE (tenant_id, segment_id, index_id) - DoUpdates: clause.AssignmentColumns([]string{"index_build_id", "enable_index", "create_time"}), - }).CreateInBatches(in, 100).Error - - if err != nil { - log.Error("upsert segment_indexes failed", zap.Error(err)) - return err - } - - return nil -} - -func (s *segmentIndexDb) MarkDeleted(tenantID string, segIndexes []*dbmodel.SegmentIndex) error { - inValues := make([][]interface{}, 0, len(segIndexes)) - for _, segIdx := range segIndexes { - in := []interface{}{segIdx.SegmentID, segIdx.IndexID} - inValues = append(inValues, in) - } - - err := s.db.Model(&dbmodel.SegmentIndex{}).Where("tenant_id = ? AND (segment_id, index_id) IN ?", tenantID, inValues).Updates(dbmodel.SegmentIndex{ - IsDeleted: true, - }).Error - - if err != nil { - log.Error("update segment_indexes deleted failed", zap.String("tenant", tenantID), zap.Any("segmentIDIndexID", inValues), zap.Error(err)) - return err - } - - return nil -} - -func (s *segmentIndexDb) MarkDeletedByCollectionID(tenantID string, collID typeutil.UniqueID) error { - err := s.db.Model(&dbmodel.SegmentIndex{}).Where("tenant_id = ? AND collection_id = ?", tenantID, collID).Updates(dbmodel.SegmentIndex{ - IsDeleted: true, - }).Error - - if err != nil { - log.Error("update segment_indexes deleted by collection id failed", zap.String("tenant", tenantID), zap.Int64("collectionID", collID), zap.Error(err)) - return err - } - - return nil -} - -func (s *segmentIndexDb) MarkDeletedByBuildID(tenantID string, buildID typeutil.UniqueID) error { - err := s.db.Model(&dbmodel.SegmentIndex{}).Where("tenant_id = ? AND build_id = ?", tenantID, buildID).Updates(dbmodel.SegmentIndex{ - IsDeleted: true, - }).Error - - if err != nil { - log.Error("update segment_indexes deleted by index id failed", zap.String("tenant", tenantID), zap.Int64("buildID", buildID), zap.Error(err)) - return err - } - - return nil -} diff --git a/internal/metastore/db/dao/segment_index_test.go b/internal/metastore/db/dao/segment_index_test.go deleted file mode 100644 index 7ca7442c71..0000000000 --- a/internal/metastore/db/dao/segment_index_test.go +++ /dev/null @@ -1,248 +0,0 @@ -package dao - -import ( - "testing" - "time" - - "github.com/cockroachdb/errors" - - "github.com/DATA-DOG/go-sqlmock" - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - "github.com/stretchr/testify/assert" -) - -func TestSegmentIndex_Insert(t *testing.T) { - var segIndexes = []*dbmodel.SegmentIndex{ - { - TenantID: tenantID, - CollectionID: collID1, - PartitionID: partitionID1, - SegmentID: segmentID1, - NumRows: NumRows, - IndexID: indexID1, - BuildID: 1002, - NodeID: 3, - IndexVersion: 0, - IndexState: 3, - FailReason: "", - CreateTime: uint64(1011), - IndexFileKeys: "", - IndexSize: 1024, - IsDeleted: false, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - }, - } - - // expectation - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `segment_indexes` (`tenant_id`,`collection_id`,`partition_id`,`segment_id`,`num_rows`,`index_id`,`build_id`,`node_id`,`index_version`,`index_state`,`fail_reason`,`create_time`,`index_file_keys`,`index_size`,`is_deleted`,`created_at`,`updated_at`) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"). - WithArgs(segIndexes[0].TenantID, segIndexes[0].CollectionID, segIndexes[0].PartitionID, segIndexes[0].SegmentID, segIndexes[0].NumRows, segIndexes[0].IndexID, segIndexes[0].BuildID, segIndexes[0].NodeID, segIndexes[0].IndexVersion, segIndexes[0].IndexState, segIndexes[0].FailReason, segIndexes[0].CreateTime, segIndexes[0].IndexFileKeys, segIndexes[0].IndexSize, segIndexes[0].IsDeleted, segIndexes[0].CreatedAt, segIndexes[0].UpdatedAt). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - - // actual - err := segIndexTestDb.Insert(segIndexes) - assert.NoError(t, err) -} - -func TestSegmentIndex_Insert_Error(t *testing.T) { - var segIndexes = []*dbmodel.SegmentIndex{ - { - TenantID: tenantID, - CollectionID: collID1, - PartitionID: partitionID1, - SegmentID: segmentID1, - NumRows: NumRows, - IndexID: indexID1, - BuildID: 1002, - NodeID: 3, - IndexVersion: 0, - IndexState: 3, - FailReason: "", - CreateTime: uint64(1011), - IndexFileKeys: "", - IndexSize: 1024, - IsDeleted: false, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - }, - } - - // expectation - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `segment_indexes` (`tenant_id`,`collection_id`,`partition_id`,`segment_id`,`num_rows`,`index_id`,`build_id`,`node_id`,`index_version`,`index_state`,`fail_reason`,`create_time`,`index_file_keys`,`index_size`,`is_deleted`,`created_at`,`updated_at`) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"). - WithArgs(segIndexes[0].TenantID, segIndexes[0].CollectionID, segIndexes[0].PartitionID, segIndexes[0].SegmentID, segIndexes[0].NumRows, segIndexes[0].IndexID, segIndexes[0].BuildID, segIndexes[0].NodeID, segIndexes[0].IndexVersion, segIndexes[0].IndexState, segIndexes[0].FailReason, segIndexes[0].CreateTime, segIndexes[0].IndexFileKeys, segIndexes[0].IndexSize, segIndexes[0].IsDeleted, segIndexes[0].CreatedAt, segIndexes[0].UpdatedAt). - WillReturnError(errors.New("test error")) - mock.ExpectRollback() - - // actual - err := segIndexTestDb.Insert(segIndexes) - assert.Error(t, err) -} - -func TestSegmentIndex_Update(t *testing.T) { - var segIndexes = []*dbmodel.SegmentIndex{ - { - TenantID: tenantID, - CollectionID: collID1, - PartitionID: partitionID1, - SegmentID: segmentID1, - NumRows: NumRows, - IndexID: indexID1, - BuildID: 1002, - NodeID: 3, - IndexVersion: 0, - IndexState: 3, - FailReason: "", - CreateTime: uint64(1011), - IndexFileKeys: "", - IndexSize: 1024, - IsDeleted: true, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - }, - } - - // expectation - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `segment_indexes` (`tenant_id`,`collection_id`,`partition_id`,`segment_id`,`num_rows`,`index_id`,`build_id`,`node_id`,`index_version`,`index_state`,`fail_reason`,`create_time`,`index_file_keys`,`index_size`,`is_deleted`,`created_at`,`updated_at`) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"). - WithArgs(segIndexes[0].TenantID, segIndexes[0].CollectionID, segIndexes[0].PartitionID, segIndexes[0].SegmentID, segIndexes[0].NumRows, segIndexes[0].IndexID, segIndexes[0].BuildID, segIndexes[0].NodeID, segIndexes[0].IndexVersion, segIndexes[0].IndexState, segIndexes[0].FailReason, segIndexes[0].CreateTime, segIndexes[0].IndexFileKeys, segIndexes[0].IndexSize, segIndexes[0].IsDeleted, segIndexes[0].CreatedAt, segIndexes[0].UpdatedAt). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - - // actual - err := segIndexTestDb.Update(segIndexes[0]) - assert.NoError(t, err) -} - -func TestSegmentIndex_Upsert_Error(t *testing.T) { - var segIndexes = []*dbmodel.SegmentIndex{ - { - TenantID: tenantID, - CollectionID: collID1, - PartitionID: partitionID1, - SegmentID: segmentID1, - NumRows: NumRows, - IndexID: indexID1, - BuildID: 1002, - NodeID: 3, - IndexVersion: 0, - IndexState: 3, - FailReason: "", - CreateTime: uint64(1011), - IndexFileKeys: "", - IndexSize: 1024, - IsDeleted: true, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - }, - } - - // expectation - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `segment_indexes` (`tenant_id`,`collection_id`,`partition_id`,`segment_id`,`num_rows`,`index_id`,`build_id`,`node_id`,`index_version`,`index_state`,`fail_reason`,`create_time`,`index_file_keys`,`index_size`,`is_deleted`,`created_at`,`updated_at`) VALUES (?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?,?)"). - WithArgs(segIndexes[0].TenantID, segIndexes[0].CollectionID, segIndexes[0].PartitionID, segIndexes[0].SegmentID, segIndexes[0].NumRows, segIndexes[0].IndexID, segIndexes[0].BuildID, segIndexes[0].NodeID, segIndexes[0].IndexVersion, segIndexes[0].IndexState, segIndexes[0].FailReason, segIndexes[0].CreateTime, segIndexes[0].IndexFileKeys, segIndexes[0].IndexSize, segIndexes[0].IsDeleted, segIndexes[0].CreatedAt, segIndexes[0].UpdatedAt). - WillReturnError(errors.New("test error")) - mock.ExpectRollback() - - // actual - err := segIndexTestDb.Update(segIndexes[0]) - assert.Error(t, err) -} - -func TestSegmentIndex_MarkDeleted(t *testing.T) { - var segIndexes = []*dbmodel.SegmentIndex{ - { - SegmentID: segmentID1, - IndexID: indexID1, - }, - { - SegmentID: segmentID2, - IndexID: indexID2, - }, - } - - // expectation - mock.ExpectBegin() - mock.ExpectExec("UPDATE `segment_indexes` SET `is_deleted`=?,`updated_at`=? WHERE tenant_id = ? AND (segment_id, index_id) IN ((?,?),(?,?))"). - WithArgs(true, AnyTime{}, tenantID, segIndexes[0].SegmentID, segIndexes[0].IndexID, segIndexes[1].SegmentID, segIndexes[1].IndexID). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - - // actual - err := segIndexTestDb.MarkDeleted(tenantID, segIndexes) - assert.NoError(t, err) -} - -func TestSegmentIndex_MarkDeleted_Error(t *testing.T) { - var segIndexes = []*dbmodel.SegmentIndex{ - { - SegmentID: segmentID1, - IndexID: indexID1, - }, - } - - // expectation - mock.ExpectBegin() - mock.ExpectExec("UPDATE `segment_indexes` SET `is_deleted`=?,`updated_at`=? WHERE tenant_id = ? AND (segment_id, index_id) IN ((?,?))"). - WithArgs(true, AnyTime{}, tenantID, segIndexes[0].SegmentID, segIndexes[0].IndexID). - WillReturnError(errors.New("test error")) - mock.ExpectRollback() - - // actual - err := segIndexTestDb.MarkDeleted(tenantID, segIndexes) - assert.Error(t, err) -} - -func TestSegmentIndex_MarkDeletedByCollID(t *testing.T) { - // expectation - mock.ExpectBegin() - mock.ExpectExec("UPDATE `segment_indexes` SET `is_deleted`=?,`updated_at`=? WHERE tenant_id = ? AND collection_id = ?"). - WithArgs(true, AnyTime{}, tenantID, collID1). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - - // actual - err := segIndexTestDb.MarkDeletedByCollectionID(tenantID, collID1) - assert.NoError(t, err) -} - -func TestSegmentIndex_MarkDeletedByCollID_Error(t *testing.T) { - // expectation - mock.ExpectBegin() - mock.ExpectExec("UPDATE `segment_indexes` SET `is_deleted`=?,`updated_at`=? WHERE tenant_id = ? AND collection_id = ?"). - WithArgs(true, AnyTime{}, tenantID, collID1). - WillReturnError(errors.New("test error")) - mock.ExpectRollback() - - // actual - err := segIndexTestDb.MarkDeletedByCollectionID(tenantID, collID1) - assert.Error(t, err) -} - -func TestSegmentIndex_MarkDeletedByBuildID(t *testing.T) { - // expectation - mock.ExpectBegin() - mock.ExpectExec("UPDATE `segment_indexes` SET `is_deleted`=?,`updated_at`=? WHERE tenant_id = ? AND build_id = ?"). - WithArgs(true, AnyTime{}, tenantID, indexBuildID1). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - - // actual - err := segIndexTestDb.MarkDeletedByBuildID(tenantID, indexBuildID1) - assert.NoError(t, err) -} - -func TestSegmentIndex_MarkDeletedByIdxID_Error(t *testing.T) { - // expectation - mock.ExpectBegin() - mock.ExpectExec("UPDATE `segment_indexes` SET `is_deleted`=?,`updated_at`=? WHERE tenant_id = ? AND build_id = ?"). - WithArgs(true, AnyTime{}, tenantID, indexBuildID1). - WillReturnError(errors.New("test error")) - mock.ExpectRollback() - - // actual - err := segIndexTestDb.MarkDeletedByBuildID(tenantID, indexBuildID1) - assert.Error(t, err) -} diff --git a/internal/metastore/db/dao/user.go b/internal/metastore/db/dao/user.go deleted file mode 100644 index bfe2babae3..0000000000 --- a/internal/metastore/db/dao/user.go +++ /dev/null @@ -1,75 +0,0 @@ -package dao - -import ( - "fmt" - - "github.com/cockroachdb/errors" - "go.uber.org/zap" - "gorm.io/gorm" - - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - "github.com/milvus-io/milvus/pkg/common" - "github.com/milvus-io/milvus/pkg/log" -) - -type userDb struct { - db *gorm.DB -} - -func (s *userDb) GetByUsername(tenantID string, username string) (*dbmodel.User, error) { - var r *dbmodel.User - - err := s.db.Model(&dbmodel.User{}).Where("tenant_id = ? AND username = ? AND is_deleted = false", tenantID, username).Take(&r).Error - - if errors.Is(err, gorm.ErrRecordNotFound) { - return nil, common.NewKeyNotExistError(fmt.Sprintf("%s/%s", tenantID, username)) - } - if err != nil { - log.Error("get user by username failed", zap.String("tenant", tenantID), zap.String("username", username), zap.Error(err)) - return nil, err - } - - return r, nil -} - -func (s *userDb) ListUser(tenantID string) ([]*dbmodel.User, error) { - var users []*dbmodel.User - - err := s.db.Model(&dbmodel.User{}).Where("tenant_id = ? AND is_deleted = false", tenantID).Find(&users).Error - if err != nil { - log.Error("list user failed", zap.String("tenant", tenantID), zap.Error(err)) - return nil, err - } - - return users, nil -} - -func (s *userDb) Insert(in *dbmodel.User) error { - err := s.db.Create(in).Error - if err != nil { - log.Error("insert credential_users failed", zap.String("tenant", in.TenantID), zap.String("username", in.Username), zap.Error(err)) - return err - } - - return nil -} - -func (s *userDb) MarkDeletedByUsername(tenantID string, username string) error { - err := s.db.Model(&dbmodel.User{}).Where("tenant_id = ? AND username = ?", tenantID, username).Update("is_deleted", true).Error - if err != nil { - log.Error("update credential_users is_deleted=true failed", zap.String("tenant", tenantID), zap.String("username", username), zap.Error(err)) - return err - } - - return nil -} - -func (s *userDb) UpdatePassword(tenantID string, username string, encryptedPassword string) error { - err := s.db.Model(&dbmodel.User{}).Where("tenant_id = ? AND username = ?", tenantID, username).Update("encrypted_password", encryptedPassword).Error - if err != nil { - log.Error("update password by username failed", zap.String("tenant", tenantID), zap.String("username", username), zap.Error(err)) - return err - } - - return nil -} diff --git a/internal/metastore/db/dao/user_role.go b/internal/metastore/db/dao/user_role.go deleted file mode 100644 index 03f9a47ef9..0000000000 --- a/internal/metastore/db/dao/user_role.go +++ /dev/null @@ -1,49 +0,0 @@ -package dao - -import ( - "go.uber.org/zap" - "gorm.io/gorm" - - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - "github.com/milvus-io/milvus/pkg/log" -) - -type userRoleDb struct { - db *gorm.DB -} - -func (u *userRoleDb) GetUserRoles(tenantID string, userID int64, roleID int64) ([]*dbmodel.UserRole, error) { - var ( - userRoles []*dbmodel.UserRole - err error - ) - err = u.db.Model(&dbmodel.UserRole{}). - Where(&dbmodel.UserRole{UserID: userID, RoleID: roleID}). - Where(dbmodel.GetCommonCondition(tenantID, false)). - Preload("User").Preload("Role"). - Find(&userRoles).Error - if err != nil { - log.Error("fail to get user-roles", zap.String("tenant_id", tenantID), zap.Int64("userID", userID), zap.Int64("roleID", roleID), zap.Error(err)) - return nil, err - } - return userRoles, nil -} - -func (u *userRoleDb) Insert(in *dbmodel.UserRole) error { - err := u.db.Create(in).Error - if err != nil { - log.Error("fail to insert the user-role", zap.Any("in", in), zap.Error(err)) - } - return err -} - -func (u *userRoleDb) Delete(tenantID string, userID int64, roleID int64) error { - err := u.db.Model(dbmodel.UserRole{}). - Where(&dbmodel.UserRole{UserID: userID, RoleID: roleID}). - Where(dbmodel.GetCommonCondition(tenantID, false)). - Update("is_deleted", true).Error - if err != nil { - log.Error("fail to delete the user-role", zap.String("tenant_id", tenantID), zap.Int64("userID", userID), zap.Int64("roleID", roleID), zap.Error(err)) - } - return err -} diff --git a/internal/metastore/db/dao/user_role_test.go b/internal/metastore/db/dao/user_role_test.go deleted file mode 100644 index e5245f9ac9..0000000000 --- a/internal/metastore/db/dao/user_role_test.go +++ /dev/null @@ -1,201 +0,0 @@ -package dao - -import ( - "testing" - - "github.com/cockroachdb/errors" - - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - - "github.com/DATA-DOG/go-sqlmock" - "github.com/stretchr/testify/assert" -) - -func TestUserRole_GetUserRoles(t *testing.T) { - var ( - userID1 = 1 - userID2 = 2 - roleID1 = 10 - roleID2 = 20 - userRoles []*dbmodel.UserRole - err error - ) - - // mock user and role - mock.ExpectQuery("SELECT * FROM `user_role` WHERE `is_deleted` = ? AND `tenant_id` = ?"). - WithArgs(false, tenantID). - WillReturnRows( - sqlmock.NewRows([]string{"tenant_id", "user_id", "role_id"}). - AddRow(tenantID, userID1, roleID1). - AddRow(tenantID, userID2, roleID2)) - - mock.ExpectQuery("SELECT * FROM `role` WHERE `role`.`id` IN (?,?)"). - WithArgs(roleID1, roleID2). - WillReturnRows( - sqlmock.NewRows([]string{"id", "tenant_id", "name"}). - AddRow(roleID1, tenantID, "foo1"). - AddRow(roleID2, tenantID, "foo2")) - - mock.ExpectQuery("SELECT * FROM `credential_users` WHERE `credential_users`.`id` IN (?,?)"). - WithArgs(userID1, userID2). - WillReturnRows( - sqlmock.NewRows([]string{"id", "tenant_id", "username"}). - AddRow(userID1, tenantID, "fo1"). - AddRow(userID2, tenantID, "fo2")) - - userRoles, err = userRoleTestDb.GetUserRoles(tenantID, 0, 0) - mock.MatchExpectationsInOrder(false) - assert.NoError(t, err) - assert.Equal(t, 2, len(userRoles)) - assert.Equal(t, "foo1", userRoles[0].Role.Name) - assert.Equal(t, "fo1", userRoles[0].User.Username) - - mock.ExpectQuery("SELECT * FROM `user_role` WHERE `is_deleted` = ? AND `tenant_id` = ?"). - WithArgs(false, tenantID). - WillReturnError(errors.New("test error")) - _, err = userRoleTestDb.GetUserRoles(tenantID, 0, 0) - mock.MatchExpectationsInOrder(false) - assert.Error(t, err) -} - -func TestUserRole_GetUserRolesWithUserID(t *testing.T) { - var ( - userID1 = 1 - roleID1 = 10 - roleID2 = 20 - userRoles []*dbmodel.UserRole - err error - ) - - mock.ExpectQuery("SELECT * FROM `user_role` WHERE `user_role`.`user_id` = ? AND `is_deleted` = ? AND `tenant_id` = ?"). - WithArgs(userID1, false, tenantID). - WillReturnRows( - sqlmock.NewRows([]string{"tenant_id", "user_id", "role_id"}). - AddRow(tenantID, userID1, roleID1). - AddRow(tenantID, userID1, roleID2)) - mock.ExpectQuery("SELECT * FROM `role` WHERE `role`.`id` IN (?,?)"). - WithArgs(roleID1, roleID2). - WillReturnRows( - sqlmock.NewRows([]string{"id", "tenant_id", "name"}). - AddRow(roleID1, tenantID, "foo1"). - AddRow(roleID2, tenantID, "foo2")) - mock.ExpectQuery("SELECT * FROM `credential_users` WHERE `credential_users`.`id` = ?"). - WithArgs(userID1). - WillReturnRows( - sqlmock.NewRows([]string{"id", "tenant_id", "username"}). - AddRow(userID1, tenantID, "fo1")) - - userRoles, err = userRoleTestDb.GetUserRoles(tenantID, int64(userID1), 0) - mock.MatchExpectationsInOrder(false) - assert.NoError(t, err) - assert.Equal(t, 2, len(userRoles)) - assert.Equal(t, "foo2", userRoles[1].Role.Name) - assert.Equal(t, "fo1", userRoles[0].User.Username) -} - -func TestUserRole_GetUserRolesWithRoleID(t *testing.T) { - var ( - userID1 = 1 - userID2 = 2 - roleID1 = 10 - userRoles []*dbmodel.UserRole - err error - ) - - mock.ExpectQuery("SELECT * FROM `user_role` WHERE `user_role`.`role_id` = ? AND `is_deleted` = ? AND `tenant_id` = ?"). - WithArgs(roleID1, false, tenantID). - WillReturnRows( - sqlmock.NewRows([]string{"tenant_id", "user_id", "role_id"}). - AddRow(tenantID, userID1, roleID1). - AddRow(tenantID, userID2, roleID1)) - mock.ExpectQuery("SELECT * FROM `role` WHERE `role`.`id` = ?"). - WithArgs(roleID1). - WillReturnRows( - sqlmock.NewRows([]string{"id", "tenant_id", "name"}). - AddRow(roleID1, tenantID, "foo1")) - mock.ExpectQuery("SELECT * FROM `credential_users` WHERE `credential_users`.`id` IN (?,?)"). - WithArgs(userID1, userID2). - WillReturnRows( - sqlmock.NewRows([]string{"id", "tenant_id", "username"}). - AddRow(userID1, tenantID, "fo1"). - AddRow(userID2, tenantID, "fo2")) - - userRoles, err = userRoleTestDb.GetUserRoles(tenantID, 0, int64(roleID1)) - mock.MatchExpectationsInOrder(false) - assert.NoError(t, err) - assert.Equal(t, 2, len(userRoles)) - assert.Equal(t, "foo1", userRoles[0].Role.Name) - assert.Equal(t, "fo2", userRoles[1].User.Username) -} - -func TestUserRole_Insert(t *testing.T) { - var ( - userRole *dbmodel.UserRole - err error - ) - userRole = &dbmodel.UserRole{ - Base: GetBase(), - UserID: 1, - RoleID: 1, - } - - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `user_role` (`tenant_id`,`is_deleted`,`created_at`,`updated_at`,`user_id`,`role_id`) VALUES (?,?,?,?,?,?)"). - WithArgs(userRole.TenantID, userRole.IsDeleted, userRole.CreatedAt, userRole.UpdatedAt, userRole.UserID, userRole.RoleID). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - err = userRoleTestDb.Insert(userRole) - mock.MatchExpectationsInOrder(false) - assert.NoError(t, err) -} - -func TestUserRole_InsertError(t *testing.T) { - var ( - userRole *dbmodel.UserRole - err error - ) - userRole = &dbmodel.UserRole{ - Base: GetBase(), - UserID: 1, - RoleID: 1, - } - - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `user_role` (`tenant_id`,`is_deleted`,`created_at`,`updated_at`,`user_id`,`role_id`) VALUES (?,?,?,?,?,?)"). - WithArgs(userRole.TenantID, userRole.IsDeleted, userRole.CreatedAt, userRole.UpdatedAt, userRole.UserID, userRole.RoleID). - WillReturnError(errors.New("test error")) - mock.ExpectRollback() - err = userRoleTestDb.Insert(userRole) - mock.MatchExpectationsInOrder(false) - assert.Error(t, err) -} - -func TestUserRole_Delete(t *testing.T) { - var ( - userRole *dbmodel.UserRole - getExec func() *sqlmock.ExpectedExec - err error - ) - userRole = &dbmodel.UserRole{ - Base: GetBase(), - UserID: 1, - RoleID: 1, - } - getExec = func() *sqlmock.ExpectedExec { - return mock.ExpectExec("UPDATE `user_role` SET `is_deleted`=?,`updated_at`=? WHERE `user_role`.`user_id` = ? AND `user_role`.`role_id` = ? AND `is_deleted` = ? AND `tenant_id` = ?"). - WithArgs(true, AnyTime{}, userRole.UserID, userRole.RoleID, userRole.IsDeleted, userRole.TenantID) - } - mock.ExpectBegin() - getExec().WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - err = userRoleTestDb.Delete(userRole.TenantID, userRole.UserID, userRole.RoleID) - mock.MatchExpectationsInOrder(false) - assert.NoError(t, err) - - mock.ExpectBegin() - getExec().WillReturnError(errors.New("test error")) - mock.ExpectRollback() - err = userRoleTestDb.Delete(userRole.TenantID, userRole.UserID, userRole.RoleID) - mock.MatchExpectationsInOrder(false) - assert.Error(t, err) -} diff --git a/internal/metastore/db/dao/user_test.go b/internal/metastore/db/dao/user_test.go deleted file mode 100644 index 04291f3003..0000000000 --- a/internal/metastore/db/dao/user_test.go +++ /dev/null @@ -1,212 +0,0 @@ -package dao - -import ( - "testing" - "time" - - "github.com/cockroachdb/errors" - - "github.com/DATA-DOG/go-sqlmock" - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - "github.com/stretchr/testify/assert" - "gorm.io/gorm" -) - -func TestUser_GetByUsername(t *testing.T) { - username := "test_username_1" - var user = &dbmodel.User{ - TenantID: tenantID, - Username: username, - EncryptedPassword: "xxx", - IsSuper: false, - } - - // expectation - mock.ExpectQuery("SELECT * FROM `credential_users` WHERE tenant_id = ? AND username = ? AND is_deleted = false LIMIT 1"). - WithArgs(tenantID, username). - WillReturnRows( - sqlmock.NewRows([]string{"tenant_id", "username", "encrypted_password", "is_super"}). - AddRow(user.TenantID, user.Username, user.EncryptedPassword, user.IsSuper)) - - // actual - res, err := userTestDb.GetByUsername(tenantID, username) - assert.NoError(t, err) - assert.Equal(t, user, res) -} - -func TestUser_GetByUsername_ErrRecordNotFound(t *testing.T) { - username := "test_username_1" - - // expectation - mock.ExpectQuery("SELECT * FROM `credential_users` WHERE tenant_id = ? AND username = ? AND is_deleted = false LIMIT 1"). - WithArgs(tenantID, username). - WillReturnError(gorm.ErrRecordNotFound) - - // actual - res, err := userTestDb.GetByUsername(tenantID, username) - assert.Nil(t, res) - assert.Error(t, err) -} - -func TestUser_GetByUsername_Error(t *testing.T) { - username := "test_username_1" - - // expectation - mock.ExpectQuery("SELECT * FROM `credential_users` WHERE tenant_id = ? AND username = ? AND is_deleted = false LIMIT 1"). - WithArgs(tenantID, username). - WillReturnError(errors.New("test error")) - - // actual - res, err := userTestDb.GetByUsername(tenantID, username) - assert.Nil(t, res) - assert.Error(t, err) -} - -func TestUser_ListUsername(t *testing.T) { - var ( - usernames = []string{ - "test_username_1", - "test_username_2", - } - user = &dbmodel.User{ - TenantID: tenantID, - EncryptedPassword: "xxx", - IsSuper: false, - } - ) - - // expectation - mock.ExpectQuery("SELECT * FROM `credential_users` WHERE tenant_id = ? AND is_deleted = false"). - WithArgs(tenantID). - WillReturnRows( - sqlmock.NewRows([]string{"tenant_id", "username", "encrypted_password", "is_super"}). - AddRow(user.TenantID, usernames[0], user.EncryptedPassword, user.IsSuper). - AddRow(user.TenantID, usernames[1], user.EncryptedPassword, user.IsSuper)) - - // actual - res, err := userTestDb.ListUser(tenantID) - assert.NoError(t, err) - assert.Equal(t, 2, len(res)) - assert.Equal(t, usernames[0], res[0].Username) - assert.Equal(t, usernames[1], res[1].Username) -} - -func TestUser_ListUsername_Error(t *testing.T) { - // expectation - mock.ExpectQuery("SELECT * FROM `credential_users` WHERE tenant_id = ? AND is_deleted = false"). - WithArgs(tenantID). - WillReturnError(errors.New("test error")) - - // actual - res, err := userTestDb.ListUser(tenantID) - assert.Nil(t, res) - assert.Error(t, err) -} - -func TestUser_Insert(t *testing.T) { - var user = &dbmodel.User{ - TenantID: tenantID, - Username: "test_username", - EncryptedPassword: "xxx", - IsSuper: false, - IsDeleted: false, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - } - - // expectation - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `credential_users` (`tenant_id`,`username`,`encrypted_password`,`is_super`,`is_deleted`,`created_at`,`updated_at`) VALUES (?,?,?,?,?,?,?)"). - WithArgs(user.TenantID, user.Username, user.EncryptedPassword, user.IsSuper, user.IsDeleted, user.CreatedAt, user.UpdatedAt). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - - // actual - err := userTestDb.Insert(user) - assert.NoError(t, err) -} - -func TestUser_Insert_Error(t *testing.T) { - var user = &dbmodel.User{ - TenantID: tenantID, - Username: "test_username", - EncryptedPassword: "xxx", - IsSuper: false, - IsDeleted: false, - CreatedAt: time.Now(), - UpdatedAt: time.Now(), - } - - // expectation - mock.ExpectBegin() - mock.ExpectExec("INSERT INTO `credential_users` (`tenant_id`,`username`,`encrypted_password`,`is_super`,`is_deleted`,`created_at`,`updated_at`) VALUES (?,?,?,?,?,?,?)"). - WithArgs(user.TenantID, user.Username, user.EncryptedPassword, user.IsSuper, user.IsDeleted, user.CreatedAt, user.UpdatedAt). - WillReturnError(errors.New("test error")) - mock.ExpectRollback() - - // actual - err := userTestDb.Insert(user) - assert.Error(t, err) -} - -func TestUser_MarkDeletedByUsername(t *testing.T) { - username := "test_username_1" - - // expectation - mock.ExpectBegin() - mock.ExpectExec("UPDATE `credential_users` SET `is_deleted`=?,`updated_at`=? WHERE tenant_id = ? AND username = ?"). - WithArgs(true, AnyTime{}, tenantID, username). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - - // actual - err := userTestDb.MarkDeletedByUsername(tenantID, username) - assert.NoError(t, err) -} - -func TestUser_MarkDeletedByUsername_Error(t *testing.T) { - username := "test_username_1" - - // expectation - mock.ExpectBegin() - mock.ExpectExec("UPDATE `credential_users` SET `is_deleted`=?,`updated_at`=? WHERE tenant_id = ? AND username = ?"). - WithArgs(true, AnyTime{}, tenantID, username). - WillReturnError(errors.New("test error")) - mock.ExpectRollback() - - // actual - err := userTestDb.MarkDeletedByUsername(tenantID, username) - assert.Error(t, err) -} - -func TestUser_UpdatePassword(t *testing.T) { - username := "test_username_1" - encryptedPassword := "test_encrypted_password_1" - - // expectation - mock.ExpectBegin() - mock.ExpectExec("UPDATE `credential_users` SET `encrypted_password`=?,`updated_at`=? WHERE tenant_id = ? AND username = ?"). - WithArgs(encryptedPassword, AnyTime{}, tenantID, username). - WillReturnResult(sqlmock.NewResult(1, 1)) - mock.ExpectCommit() - - // actual - err := userTestDb.UpdatePassword(tenantID, username, encryptedPassword) - assert.NoError(t, err) -} - -func TestUser_UpdatePassword_Error(t *testing.T) { - username := "test_username_1" - encryptedPassword := "test_encrypted_password_1" - - // expectation - mock.ExpectBegin() - mock.ExpectExec("UPDATE `credential_users` SET `encrypted_password`=?,`updated_at`=? WHERE tenant_id = ? AND username = ?"). - WithArgs(encryptedPassword, AnyTime{}, tenantID, username). - WillReturnError(errors.New("test error")) - mock.ExpectRollback() - - // actual - err := userTestDb.UpdatePassword(tenantID, username, encryptedPassword) - assert.Error(t, err) -} diff --git a/internal/metastore/db/dbcore/core.go b/internal/metastore/db/dbcore/core.go deleted file mode 100644 index 19ce122d48..0000000000 --- a/internal/metastore/db/dbcore/core.go +++ /dev/null @@ -1,119 +0,0 @@ -package dbcore - -import ( - "context" - "fmt" - "reflect" - - "go.uber.org/zap" - "gorm.io/driver/mysql" - "gorm.io/gorm" - "gorm.io/gorm/logger" - - "github.com/milvus-io/milvus/pkg/log" - "github.com/milvus-io/milvus/pkg/util/paramtable" -) - -var ( - globalDB *gorm.DB -) - -func Connect(cfg *paramtable.MetaDBConfig) error { - // load config - dsn := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?charset=utf8mb4&parseTime=True&loc=Local", - cfg.Username.GetValue(), cfg.Password.GetValue(), cfg.Address.GetValue(), cfg.Port.GetAsInt(), cfg.DBName.GetValue()) - - var ormLogger logger.Interface - if log.Level().String() == "debug" { - ormLogger = logger.Default.LogMode(logger.Info) - } else { - ormLogger = logger.Default - } - - db, err := gorm.Open(mysql.Open(dsn), &gorm.Config{ - Logger: ormLogger, - CreateBatchSize: 100, - }) - if err != nil { - log.Error("fail to connect db", - zap.String("host", cfg.Address.GetValue()), - zap.Int("port", cfg.Port.GetAsInt()), - zap.String("database", cfg.DBName.GetValue()), - zap.Error(err)) - return err - } - - idb, err := db.DB() - if err != nil { - log.Error("fail to create db instance", - zap.String("host", cfg.Address.GetValue()), - zap.Int("port", cfg.Port.GetAsInt()), - zap.String("database", cfg.DBName.GetValue()), - zap.Error(err)) - return err - } - idb.SetMaxIdleConns(cfg.MaxIdleConns.GetAsInt()) - idb.SetMaxOpenConns(cfg.MaxOpenConns.GetAsInt()) - - globalDB = db - - log.Info("db connected success", - zap.String("host", cfg.Address.GetValue()), - zap.Int("port", cfg.Port.GetAsInt()), - zap.String("database", cfg.DBName.GetValue()), - zap.Error(err)) - - return nil -} - -// SetGlobalDB Only for test -func SetGlobalDB(db *gorm.DB) { - globalDB = db -} - -type ctxTransactionKey struct{} - -func CtxWithTransaction(ctx context.Context, tx *gorm.DB) context.Context { - if ctx == nil { - ctx = context.Background() - } - return context.WithValue(ctx, ctxTransactionKey{}, tx) -} - -type txImpl struct{} - -func NewTxImpl() *txImpl { - return &txImpl{} -} - -func (*txImpl) Transaction(ctx context.Context, fn func(txctx context.Context) error) error { - db := globalDB.WithContext(ctx) - - return db.Transaction(func(tx *gorm.DB) error { - txCtx := CtxWithTransaction(ctx, tx) - return fn(txCtx) - }) -} - -func GetDB(ctx context.Context) *gorm.DB { - iface := ctx.Value(ctxTransactionKey{}) - - if iface != nil { - tx, ok := iface.(*gorm.DB) - if !ok { - log.Error("unexpect context value type", zap.Any("type", reflect.TypeOf(tx))) - return nil - } - - return tx - } - - return globalDB.WithContext(ctx) -} - -//type CommonModel struct { -// ID string `gorm:"primary_key"` -// IsDeleted bool `gorm:"is_deleted"` -// CreatedAt time.Time `gorm:"created_at"` -// UpdatedAt time.Time `gorm:"updated_at"` -//} diff --git a/internal/metastore/db/dbmodel/base_model.go b/internal/metastore/db/dbmodel/base_model.go deleted file mode 100644 index 5ab3fa70b9..0000000000 --- a/internal/metastore/db/dbmodel/base_model.go +++ /dev/null @@ -1,11 +0,0 @@ -package dbmodel - -import "time" - -type Base struct { - ID int64 `gorm:"id"` - TenantID string `gorm:"tenant_id"` - IsDeleted bool `gorm:"is_deleted"` - CreatedAt time.Time `gorm:"created_at"` - UpdatedAt time.Time `gorm:"updated_at"` -} diff --git a/internal/metastore/db/dbmodel/collection.go b/internal/metastore/db/dbmodel/collection.go deleted file mode 100644 index a9016c8121..0000000000 --- a/internal/metastore/db/dbmodel/collection.go +++ /dev/null @@ -1,107 +0,0 @@ -package dbmodel - -import ( - "encoding/json" - "fmt" - "time" - - "go.uber.org/zap" - - "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" - "github.com/milvus-io/milvus/internal/metastore/model" - "github.com/milvus-io/milvus/pkg/log" - "github.com/milvus-io/milvus/pkg/util/typeutil" -) - -type Collection struct { - ID int64 `gorm:"id"` - TenantID string `gorm:"tenant_id"` - CollectionID int64 `gorm:"collection_id"` - CollectionName string `gorm:"collection_name"` - Description string `gorm:"description"` - AutoID bool `gorm:"auto_id"` - ShardsNum int32 `gorm:"shards_num"` - StartPosition string `gorm:"start_position"` - ConsistencyLevel int32 `gorm:"consistency_level"` - Status int32 `gorm:"status"` - Properties string `gorm:"properties"` - Ts typeutil.Timestamp `gorm:"ts"` - IsDeleted bool `gorm:"is_deleted"` - CreatedAt time.Time `gorm:"created_at"` - UpdatedAt time.Time `gorm:"updated_at"` -} - -func (v Collection) TableName() string { - return "collections" -} - -//go:generate mockery --name=ICollectionDb -type ICollectionDb interface { - // GetCollectionIdTs get the largest timestamp that less than or equal to param ts, no matter is_deleted is true or false. - GetCollectionIDTs(tenantID string, collectionID typeutil.UniqueID, ts typeutil.Timestamp) (*Collection, error) - ListCollectionIDTs(tenantID string, ts typeutil.Timestamp) ([]*Collection, error) - Get(tenantID string, collectionID typeutil.UniqueID, ts typeutil.Timestamp) (*Collection, error) - GetCollectionIDByName(tenantID string, collectionName string, ts typeutil.Timestamp) (typeutil.UniqueID, error) - Insert(in *Collection) error - Update(in *Collection) error -} - -// model <---> db - -func UnmarshalCollectionModel(coll *Collection) (*model.Collection, error) { - var startPositions []*commonpb.KeyDataPair - if coll.StartPosition != "" { - err := json.Unmarshal([]byte(coll.StartPosition), &startPositions) - if err != nil { - log.Error("unmarshal collection start positions error", zap.Int64("collectionID", coll.CollectionID), zap.Uint64("ts", coll.Ts), zap.Error(err)) - return nil, err - } - } - - properties, err := UnmarshalProperties(coll.Properties) - if err != nil { - log.Error("unmarshal collection properties error", zap.Int64("collectionID", coll.CollectionID), - zap.String("properties", coll.Properties), zap.Error(err)) - return nil, err - } - - return &model.Collection{ - TenantID: coll.TenantID, - CollectionID: coll.CollectionID, - Name: coll.CollectionName, - Description: coll.Description, - AutoID: coll.AutoID, - ShardsNum: coll.ShardsNum, - StartPositions: startPositions, - ConsistencyLevel: commonpb.ConsistencyLevel(coll.ConsistencyLevel), - CreateTime: coll.Ts, - Properties: properties, - }, nil -} - -func UnmarshalProperties(propertiesStr string) ([]*commonpb.KeyValuePair, error) { - if propertiesStr == "" { - return nil, nil - } - - var properties []*commonpb.KeyValuePair - if propertiesStr != "" { - if err := json.Unmarshal([]byte(propertiesStr), &properties); err != nil { - return nil, fmt.Errorf("failed to unmarshal properties: %s", err.Error()) - } - } - - return properties, nil -} - -func MarshalProperties(properties []*commonpb.KeyValuePair) (string, error) { - if properties == nil { - return "", nil - } - - propertiesBytes, err := json.Marshal(properties) - if err != nil { - return "", fmt.Errorf("failed to marshal properties: %s", err.Error()) - } - return string(propertiesBytes), nil -} diff --git a/internal/metastore/db/dbmodel/collection_alias.go b/internal/metastore/db/dbmodel/collection_alias.go deleted file mode 100644 index bd6327835d..0000000000 --- a/internal/metastore/db/dbmodel/collection_alias.go +++ /dev/null @@ -1,30 +0,0 @@ -package dbmodel - -import ( - "time" - - "github.com/milvus-io/milvus/pkg/util/typeutil" -) - -type CollectionAlias struct { - ID int64 `gorm:"id"` - TenantID string `gorm:"tenant_id"` - CollectionID int64 `gorm:"collection_id"` - CollectionAlias string `gorm:"collection_alias"` - Ts typeutil.Timestamp `gorm:"ts"` - IsDeleted bool `gorm:"is_deleted"` - CreatedAt time.Time `gorm:"created_at"` - UpdatedAt time.Time `gorm:"updated_at"` -} - -func (v CollectionAlias) TableName() string { - return "collection_aliases" -} - -//go:generate mockery --name=ICollAliasDb -type ICollAliasDb interface { - Insert(in []*CollectionAlias) error - GetCollectionIDByAlias(tenantID string, alias string, ts typeutil.Timestamp) (typeutil.UniqueID, error) - ListCollectionIDTs(tenantID string, ts typeutil.Timestamp) ([]*CollectionAlias, error) - List(tenantID string, cidTsPairs []*CollectionAlias) ([]*CollectionAlias, error) -} diff --git a/internal/metastore/db/dbmodel/collection_channel.go b/internal/metastore/db/dbmodel/collection_channel.go deleted file mode 100644 index 3318f517b4..0000000000 --- a/internal/metastore/db/dbmodel/collection_channel.go +++ /dev/null @@ -1,40 +0,0 @@ -package dbmodel - -import ( - "time" - - "github.com/milvus-io/milvus/pkg/util/typeutil" -) - -type CollectionChannel struct { - ID int64 `gorm:"id"` - TenantID string `gorm:"tenant_id"` - CollectionID int64 `gorm:"collection_id"` - VirtualChannelName string `gorm:"virtual_channel_name"` - PhysicalChannelName string `gorm:"physical_channel_name"` - Removed bool `gorm:"removed"` - Ts typeutil.Timestamp `gorm:"ts"` - IsDeleted bool `gorm:"is_deleted"` - CreatedAt time.Time `gorm:"created_at"` - UpdatedAt time.Time `gorm:"updated_at"` -} - -func (v CollectionChannel) TableName() string { - return "collection_channels" -} - -//go:generate mockery --name=ICollChannelDb -type ICollChannelDb interface { - GetByCollectionID(tenantID string, collectionID typeutil.UniqueID, ts typeutil.Timestamp) ([]*CollectionChannel, error) - Insert(in []*CollectionChannel) error -} - -func ExtractChannelNames(channels []*CollectionChannel) ([]string, []string) { - vchans := make([]string, 0, len(channels)) - pchans := make([]string, 0, len(channels)) - for _, ch := range channels { - vchans = append(vchans, ch.VirtualChannelName) - pchans = append(pchans, ch.PhysicalChannelName) - } - return vchans, pchans -} diff --git a/internal/metastore/db/dbmodel/collection_test.go b/internal/metastore/db/dbmodel/collection_test.go deleted file mode 100644 index 91f9cb4589..0000000000 --- a/internal/metastore/db/dbmodel/collection_test.go +++ /dev/null @@ -1,106 +0,0 @@ -package dbmodel - -import ( - "testing" - "time" - - "github.com/stretchr/testify/assert" - - "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" - "github.com/milvus-io/milvus/pkg/common" -) - -var ( - ts = time.Now() -) - -func TestUnmarshalCollectionModel(t *testing.T) { - t.Run("Unmarshal start position fail", func(t *testing.T) { - collection := &Collection{ - StartPosition: "{\"error json\":}", - } - - ret, err := UnmarshalCollectionModel(collection) - assert.Nil(t, ret) - assert.Error(t, err) - }) - - t.Run("Unmarshal properties fail", func(t *testing.T) { - collection := &Collection{ - Properties: "{\"error json\":}", - } - - ret, err := UnmarshalCollectionModel(collection) - assert.Nil(t, ret) - assert.Error(t, err) - }) - - t.Run("Unmarshal collection successfully", func(t *testing.T) { - collection := &Collection{ - TenantID: "", - CollectionID: 1, - CollectionName: "cn", - Description: "", - AutoID: false, - ShardsNum: common.DefaultShardsNum, - StartPosition: "", - ConsistencyLevel: int32(commonpb.ConsistencyLevel_Eventually), - Properties: "", - Ts: 1, - IsDeleted: false, - CreatedAt: ts, - UpdatedAt: ts, - } - - ret, err := UnmarshalCollectionModel(collection) - assert.NotNil(t, ret) - assert.NoError(t, err) - - assert.Equal(t, "", ret.TenantID) - assert.Equal(t, int64(1), ret.CollectionID) - assert.Equal(t, "cn", ret.Name) - assert.Equal(t, "", ret.Description) - assert.Equal(t, false, ret.AutoID) - assert.Equal(t, common.DefaultShardsNum, ret.ShardsNum) - assert.Equal(t, 0, len(ret.StartPositions)) - assert.Equal(t, commonpb.ConsistencyLevel(3), ret.ConsistencyLevel) - assert.Nil(t, ret.Properties) - assert.Equal(t, uint64(1), ret.CreateTime) - }) -} - -func TestUnmarshalAndMarshalProperties(t *testing.T) { - t.Run("Unmarshal and Marshal empty", func(t *testing.T) { - - ret, err := UnmarshalProperties("") - assert.Nil(t, ret) - assert.NoError(t, err) - - ret2, err := MarshalProperties(nil) - assert.Empty(t, ret2) - assert.NoError(t, err) - }) - - t.Run("Unmarshal and Marshal fail", func(t *testing.T) { - ret, err := UnmarshalProperties("{\"error json\":}") - assert.Nil(t, ret) - assert.Error(t, err) - }) - - t.Run("Unmarshal collection successfully", func(t *testing.T) { - properties := []*commonpb.KeyValuePair{ - { - Key: common.CollectionTTLConfigKey, - Value: "3600", - }, - } - propertiesStr, err := MarshalProperties(properties) - assert.NotEmpty(t, propertiesStr) - assert.NoError(t, err) - - ret2, err := UnmarshalProperties(propertiesStr) - assert.NotNil(t, ret2) - assert.NoError(t, err) - assert.Equal(t, ret2, properties) - }) -} diff --git a/internal/metastore/db/dbmodel/common.go b/internal/metastore/db/dbmodel/common.go deleted file mode 100644 index 60c5dd4b55..0000000000 --- a/internal/metastore/db/dbmodel/common.go +++ /dev/null @@ -1,30 +0,0 @@ -package dbmodel - -import "context" - -//go:generate mockery --name=IMetaDomain -type IMetaDomain interface { - CollectionDb(ctx context.Context) ICollectionDb - FieldDb(ctx context.Context) IFieldDb - CollChannelDb(ctx context.Context) ICollChannelDb - CollAliasDb(ctx context.Context) ICollAliasDb - PartitionDb(ctx context.Context) IPartitionDb - IndexDb(ctx context.Context) IIndexDb - SegmentIndexDb(ctx context.Context) ISegmentIndexDb - UserDb(ctx context.Context) IUserDb - RoleDb(ctx context.Context) IRoleDb - UserRoleDb(ctx context.Context) IUserRoleDb - GrantDb(ctx context.Context) IGrantDb - GrantIDDb(ctx context.Context) IGrantIDDb -} - -type ITransaction interface { - Transaction(ctx context.Context, fn func(txCtx context.Context) error) error -} - -func GetCommonCondition(tenant string, isDelete bool) map[string]interface{} { - return map[string]interface{}{ - "tenant_id": tenant, - "is_deleted": isDelete, - } -} diff --git a/internal/metastore/db/dbmodel/field.go b/internal/metastore/db/dbmodel/field.go deleted file mode 100644 index 1fa73c04ca..0000000000 --- a/internal/metastore/db/dbmodel/field.go +++ /dev/null @@ -1,91 +0,0 @@ -package dbmodel - -import ( - "encoding/json" - "time" - - "go.uber.org/zap" - - "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" - "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" - "github.com/milvus-io/milvus/internal/metastore/model" - "github.com/milvus-io/milvus/pkg/log" - "github.com/milvus-io/milvus/pkg/util/funcutil" - "github.com/milvus-io/milvus/pkg/util/typeutil" -) - -type Field struct { - ID int64 `gorm:"id"` - TenantID string `gorm:"tenant_id"` - FieldID int64 `gorm:"field_id"` - FieldName string `gorm:"field_name"` - IsPrimaryKey bool `gorm:"is_primary_key"` - Description string `gorm:"description"` - DataType schemapb.DataType `gorm:"data_type"` - TypeParams string `gorm:"type_params"` - IndexParams string `gorm:"index_params"` - AutoID bool `gorm:"auto_id"` - CollectionID int64 `gorm:"collection_id"` - Ts typeutil.Timestamp `gorm:"ts"` - IsDeleted bool `gorm:"is_deleted"` - CreatedAt time.Time `gorm:"created_at"` - UpdatedAt time.Time `gorm:"updated_at"` -} - -func (v Field) TableName() string { - return "field_schemas" -} - -//go:generate mockery --name=IFieldDb -type IFieldDb interface { - GetByCollectionID(tenantID string, collectionID typeutil.UniqueID, ts typeutil.Timestamp) ([]*Field, error) - Insert(in []*Field) error -} - -// model <---> db - -func UnmarshalFieldModel(fields []*Field) ([]*model.Field, error) { - r := make([]*model.Field, 0, len(fields)) - for _, f := range fields { - fd, err := ConvertFieldDBToModel(f) - if err != nil { - return nil, err - } - r = append(r, fd) - } - - return r, nil -} - -func ConvertFieldDBToModel(field *Field) (*model.Field, error) { - var typeParams []commonpb.KeyValuePair - if field.TypeParams != "" { - err := json.Unmarshal([]byte(field.TypeParams), &typeParams) - if err != nil { - log.Error("unmarshal TypeParams of field failed", zap.Int64("collectionID", field.CollectionID), - zap.Int64("fieldID", field.FieldID), zap.String("fieldName", field.FieldName), zap.Error(err)) - return nil, err - } - } - - var indexParams []commonpb.KeyValuePair - if field.IndexParams != "" { - err := json.Unmarshal([]byte(field.IndexParams), &indexParams) - if err != nil { - log.Error("unmarshal IndexParams of field failed", zap.Int64("collectionID", field.CollectionID), - zap.Int64("fieldID", field.FieldID), zap.String("fieldName", field.FieldName), zap.Error(err)) - return nil, err - } - } - - return &model.Field{ - FieldID: field.FieldID, - Name: field.FieldName, - IsPrimaryKey: field.IsPrimaryKey, - Description: field.Description, - DataType: field.DataType, - TypeParams: funcutil.ConvertToKeyValuePairPointer(typeParams), - IndexParams: funcutil.ConvertToKeyValuePairPointer(indexParams), - AutoID: field.AutoID, - }, nil -} diff --git a/internal/metastore/db/dbmodel/grant.go b/internal/metastore/db/dbmodel/grant.go deleted file mode 100644 index ffb9e6af30..0000000000 --- a/internal/metastore/db/dbmodel/grant.go +++ /dev/null @@ -1,20 +0,0 @@ -package dbmodel - -type Grant struct { - Base - RoleID int64 `gorm:"role_id"` - Role Role `gorm:"foreignKey:RoleID"` - Object string `gorm:"object"` - ObjectName string `gorm:"object_name"` -} - -func (g *Grant) TableName() string { - return "grant" -} - -//go:generate mockery --name=IGrantDb -type IGrantDb interface { - GetGrants(tenantID string, roleID int64, object string, objectName string) ([]*Grant, error) - Insert(in *Grant) error - Delete(tenantID string, roleID int64, object string, objectName string) error -} diff --git a/internal/metastore/db/dbmodel/grant_id.go b/internal/metastore/db/dbmodel/grant_id.go deleted file mode 100644 index d67e00eba7..0000000000 --- a/internal/metastore/db/dbmodel/grant_id.go +++ /dev/null @@ -1,21 +0,0 @@ -package dbmodel - -type GrantID struct { - Base - GrantID int64 `gorm:"grant_id"` - Grant Grant `gorm:"foreignKey:GrantID"` - Privilege string `gorm:"privilege"` - GrantorID int64 `gorm:"grantor_id"` - Grantor User `gorm:"foreignKey:GrantorID"` -} - -func (g *GrantID) TableName() string { - return "grant_id" -} - -//go:generate mockery --name=IGrantIDDb -type IGrantIDDb interface { - GetGrantIDs(tenantID string, grantID int64, privilege string, preloadGrant bool, preloadGrantor bool) ([]*GrantID, error) - Insert(in *GrantID) error - Delete(tenantID string, grantID int64, privilege string) error -} diff --git a/internal/metastore/db/dbmodel/index.go b/internal/metastore/db/dbmodel/index.go deleted file mode 100644 index ca8663ae1c..0000000000 --- a/internal/metastore/db/dbmodel/index.go +++ /dev/null @@ -1,127 +0,0 @@ -package dbmodel - -import ( - "encoding/json" - "time" - - "go.uber.org/zap" - - "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" - "github.com/milvus-io/milvus/internal/metastore/model" - "github.com/milvus-io/milvus/pkg/common" - "github.com/milvus-io/milvus/pkg/log" - "github.com/milvus-io/milvus/pkg/util/funcutil" - "github.com/milvus-io/milvus/pkg/util/typeutil" -) - -type Index struct { - ID int64 `gorm:"id"` - TenantID string `gorm:"tenant_id"` - FieldID int64 `gorm:"field_id"` - CollectionID int64 `gorm:"collection_id"` - IndexID int64 `gorm:"index_id"` - IndexName string `gorm:"index_name"` - IndexParams string `gorm:"index_params"` - TypeParams string `gorm:"type_params"` - UserIndexParams string `gorm:"user_index_params"` - IsAutoIndex bool `gorm:"is_auto_index"` - CreateTime uint64 `gorm:"create_time"` - IsDeleted bool `gorm:"is_deleted"` - CreatedAt time.Time `gorm:"created_at"` - UpdatedAt time.Time `gorm:"updated_at"` -} - -func (v Index) TableName() string { - return "indexes" -} - -// ------------- search result ------------- - -type IndexResult struct { - FieldID int64 - CollectionID int64 - IndexID int64 - IndexName string - TypeParams string - IndexParams string - CreateTime uint64 - IsDeleted bool - IsAutoIndex bool - UserIndexParams string -} - -//go:generate mockery --name=IIndexDb -type IIndexDb interface { - Get(tenantID string, collectionID typeutil.UniqueID) ([]*Index, error) - List(tenantID string) ([]*IndexResult, error) - Insert(in []*Index) error - Update(in *Index) error - MarkDeletedByCollectionID(tenantID string, collID typeutil.UniqueID) error - MarkDeletedByIndexID(tenantID string, idxID typeutil.UniqueID) error -} - -// model <---> db - -func UnmarshalIndexModel(inputs []*IndexResult) ([]*model.Index, error) { - result := make([]*model.Index, 0, len(inputs)) - for _, ir := range inputs { - var indexParams []commonpb.KeyValuePair - if ir.IndexParams != "" { - err := json.Unmarshal([]byte(ir.IndexParams), &indexParams) - if err != nil { - log.Error("unmarshal IndexParams of index failed", zap.Int64("collectionID", ir.CollectionID), - zap.Int64("indexID", ir.IndexID), zap.String("indexName", ir.IndexName), zap.Error(err)) - return nil, err - } - } - var userIndexParams []commonpb.KeyValuePair - if ir.UserIndexParams != "" { - err := json.Unmarshal([]byte(ir.UserIndexParams), &userIndexParams) - if err != nil { - log.Error("unmarshal UserIndexParams of index failed", zap.Int64("collectionID", ir.CollectionID), - zap.Int64("indexID", ir.IndexID), zap.String("indexName", ir.IndexName), zap.Error(err)) - return nil, err - } - } - - var typeParams []commonpb.KeyValuePair - if ir.TypeParams != "" { - err := json.Unmarshal([]byte(ir.TypeParams), &typeParams) - if err != nil { - log.Error("unmarshal TypeParams of index failed", zap.Int64("collectionID", ir.CollectionID), - zap.Int64("indexID", ir.IndexID), zap.String("indexName", ir.IndexName), zap.Error(err)) - return nil, err - } - } - - idx := &model.Index{ - CollectionID: ir.CollectionID, - FieldID: ir.FieldID, - IndexID: ir.IndexID, - IndexName: ir.IndexName, - IndexParams: funcutil.ConvertToKeyValuePairPointer(indexParams), - TypeParams: funcutil.ConvertToKeyValuePairPointer(typeParams), - CreateTime: ir.CreateTime, - IsDeleted: ir.IsDeleted, - IsAutoIndex: ir.IsAutoIndex, - UserIndexParams: funcutil.ConvertToKeyValuePairPointer(userIndexParams), - } - result = append(result, idx) - } - - return result, nil -} - -func ConvertIndexDBToModel(indexes []*Index) []common.Int64Tuple { - r := make([]common.Int64Tuple, 0, len(indexes)) - - for _, idx := range indexes { - tuple := common.Int64Tuple{ - Key: idx.FieldID, - Value: idx.IndexID, - } - r = append(r, tuple) - } - - return r -} diff --git a/internal/metastore/db/dbmodel/mocks/ICollAliasDb.go b/internal/metastore/db/dbmodel/mocks/ICollAliasDb.go deleted file mode 100644 index caab2ed344..0000000000 --- a/internal/metastore/db/dbmodel/mocks/ICollAliasDb.go +++ /dev/null @@ -1,109 +0,0 @@ -// Code generated by mockery v2.14.0. DO NOT EDIT. - -package mocks - -import ( - dbmodel "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - mock "github.com/stretchr/testify/mock" -) - -// ICollAliasDb is an autogenerated mock type for the ICollAliasDb type -type ICollAliasDb struct { - mock.Mock -} - -// GetCollectionIDByAlias provides a mock function with given fields: tenantID, alias, ts -func (_m *ICollAliasDb) GetCollectionIDByAlias(tenantID string, alias string, ts uint64) (int64, error) { - ret := _m.Called(tenantID, alias, ts) - - var r0 int64 - if rf, ok := ret.Get(0).(func(string, string, uint64) int64); ok { - r0 = rf(tenantID, alias, ts) - } else { - r0 = ret.Get(0).(int64) - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, string, uint64) error); ok { - r1 = rf(tenantID, alias, ts) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Insert provides a mock function with given fields: in -func (_m *ICollAliasDb) Insert(in []*dbmodel.CollectionAlias) error { - ret := _m.Called(in) - - var r0 error - if rf, ok := ret.Get(0).(func([]*dbmodel.CollectionAlias) error); ok { - r0 = rf(in) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// List provides a mock function with given fields: tenantID, cidTsPairs -func (_m *ICollAliasDb) List(tenantID string, cidTsPairs []*dbmodel.CollectionAlias) ([]*dbmodel.CollectionAlias, error) { - ret := _m.Called(tenantID, cidTsPairs) - - var r0 []*dbmodel.CollectionAlias - if rf, ok := ret.Get(0).(func(string, []*dbmodel.CollectionAlias) []*dbmodel.CollectionAlias); ok { - r0 = rf(tenantID, cidTsPairs) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*dbmodel.CollectionAlias) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, []*dbmodel.CollectionAlias) error); ok { - r1 = rf(tenantID, cidTsPairs) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// ListCollectionIDTs provides a mock function with given fields: tenantID, ts -func (_m *ICollAliasDb) ListCollectionIDTs(tenantID string, ts uint64) ([]*dbmodel.CollectionAlias, error) { - ret := _m.Called(tenantID, ts) - - var r0 []*dbmodel.CollectionAlias - if rf, ok := ret.Get(0).(func(string, uint64) []*dbmodel.CollectionAlias); ok { - r0 = rf(tenantID, ts) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*dbmodel.CollectionAlias) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, uint64) error); ok { - r1 = rf(tenantID, ts) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -type mockConstructorTestingTNewICollAliasDb interface { - mock.TestingT - Cleanup(func()) -} - -// NewICollAliasDb creates a new instance of ICollAliasDb. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewICollAliasDb(t mockConstructorTestingTNewICollAliasDb) *ICollAliasDb { - mock := &ICollAliasDb{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/internal/metastore/db/dbmodel/mocks/ICollChannelDb.go b/internal/metastore/db/dbmodel/mocks/ICollChannelDb.go deleted file mode 100644 index 2110769067..0000000000 --- a/internal/metastore/db/dbmodel/mocks/ICollChannelDb.go +++ /dev/null @@ -1,65 +0,0 @@ -// Code generated by mockery v2.14.0. DO NOT EDIT. - -package mocks - -import ( - dbmodel "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - mock "github.com/stretchr/testify/mock" -) - -// ICollChannelDb is an autogenerated mock type for the ICollChannelDb type -type ICollChannelDb struct { - mock.Mock -} - -// GetByCollectionID provides a mock function with given fields: tenantID, collectionID, ts -func (_m *ICollChannelDb) GetByCollectionID(tenantID string, collectionID int64, ts uint64) ([]*dbmodel.CollectionChannel, error) { - ret := _m.Called(tenantID, collectionID, ts) - - var r0 []*dbmodel.CollectionChannel - if rf, ok := ret.Get(0).(func(string, int64, uint64) []*dbmodel.CollectionChannel); ok { - r0 = rf(tenantID, collectionID, ts) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*dbmodel.CollectionChannel) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, int64, uint64) error); ok { - r1 = rf(tenantID, collectionID, ts) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Insert provides a mock function with given fields: in -func (_m *ICollChannelDb) Insert(in []*dbmodel.CollectionChannel) error { - ret := _m.Called(in) - - var r0 error - if rf, ok := ret.Get(0).(func([]*dbmodel.CollectionChannel) error); ok { - r0 = rf(in) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -type mockConstructorTestingTNewICollChannelDb interface { - mock.TestingT - Cleanup(func()) -} - -// NewICollChannelDb creates a new instance of ICollChannelDb. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewICollChannelDb(t mockConstructorTestingTNewICollChannelDb) *ICollChannelDb { - mock := &ICollChannelDb{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/internal/metastore/db/dbmodel/mocks/ICollectionDb.go b/internal/metastore/db/dbmodel/mocks/ICollectionDb.go deleted file mode 100644 index 18aec42029..0000000000 --- a/internal/metastore/db/dbmodel/mocks/ICollectionDb.go +++ /dev/null @@ -1,146 +0,0 @@ -// Code generated by mockery v2.14.0. DO NOT EDIT. - -package mocks - -import ( - dbmodel "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - mock "github.com/stretchr/testify/mock" -) - -// ICollectionDb is an autogenerated mock type for the ICollectionDb type -type ICollectionDb struct { - mock.Mock -} - -// Get provides a mock function with given fields: tenantID, collectionID, ts -func (_m *ICollectionDb) Get(tenantID string, collectionID int64, ts uint64) (*dbmodel.Collection, error) { - ret := _m.Called(tenantID, collectionID, ts) - - var r0 *dbmodel.Collection - if rf, ok := ret.Get(0).(func(string, int64, uint64) *dbmodel.Collection); ok { - r0 = rf(tenantID, collectionID, ts) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*dbmodel.Collection) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, int64, uint64) error); ok { - r1 = rf(tenantID, collectionID, ts) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetCollectionIDByName provides a mock function with given fields: tenantID, collectionName, ts -func (_m *ICollectionDb) GetCollectionIDByName(tenantID string, collectionName string, ts uint64) (int64, error) { - ret := _m.Called(tenantID, collectionName, ts) - - var r0 int64 - if rf, ok := ret.Get(0).(func(string, string, uint64) int64); ok { - r0 = rf(tenantID, collectionName, ts) - } else { - r0 = ret.Get(0).(int64) - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, string, uint64) error); ok { - r1 = rf(tenantID, collectionName, ts) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// GetCollectionIDTs provides a mock function with given fields: tenantID, collectionID, ts -func (_m *ICollectionDb) GetCollectionIDTs(tenantID string, collectionID int64, ts uint64) (*dbmodel.Collection, error) { - ret := _m.Called(tenantID, collectionID, ts) - - var r0 *dbmodel.Collection - if rf, ok := ret.Get(0).(func(string, int64, uint64) *dbmodel.Collection); ok { - r0 = rf(tenantID, collectionID, ts) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*dbmodel.Collection) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, int64, uint64) error); ok { - r1 = rf(tenantID, collectionID, ts) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Insert provides a mock function with given fields: in -func (_m *ICollectionDb) Insert(in *dbmodel.Collection) error { - ret := _m.Called(in) - - var r0 error - if rf, ok := ret.Get(0).(func(*dbmodel.Collection) error); ok { - r0 = rf(in) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// ListCollectionIDTs provides a mock function with given fields: tenantID, ts -func (_m *ICollectionDb) ListCollectionIDTs(tenantID string, ts uint64) ([]*dbmodel.Collection, error) { - ret := _m.Called(tenantID, ts) - - var r0 []*dbmodel.Collection - if rf, ok := ret.Get(0).(func(string, uint64) []*dbmodel.Collection); ok { - r0 = rf(tenantID, ts) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*dbmodel.Collection) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, uint64) error); ok { - r1 = rf(tenantID, ts) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Update provides a mock function with given fields: in -func (_m *ICollectionDb) Update(in *dbmodel.Collection) error { - ret := _m.Called(in) - - var r0 error - if rf, ok := ret.Get(0).(func(*dbmodel.Collection) error); ok { - r0 = rf(in) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -type mockConstructorTestingTNewICollectionDb interface { - mock.TestingT - Cleanup(func()) -} - -// NewICollectionDb creates a new instance of ICollectionDb. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewICollectionDb(t mockConstructorTestingTNewICollectionDb) *ICollectionDb { - mock := &ICollectionDb{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/internal/metastore/db/dbmodel/mocks/IFieldDb.go b/internal/metastore/db/dbmodel/mocks/IFieldDb.go deleted file mode 100644 index 9e10bf3fe8..0000000000 --- a/internal/metastore/db/dbmodel/mocks/IFieldDb.go +++ /dev/null @@ -1,65 +0,0 @@ -// Code generated by mockery v2.14.0. DO NOT EDIT. - -package mocks - -import ( - dbmodel "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - mock "github.com/stretchr/testify/mock" -) - -// IFieldDb is an autogenerated mock type for the IFieldDb type -type IFieldDb struct { - mock.Mock -} - -// GetByCollectionID provides a mock function with given fields: tenantID, collectionID, ts -func (_m *IFieldDb) GetByCollectionID(tenantID string, collectionID int64, ts uint64) ([]*dbmodel.Field, error) { - ret := _m.Called(tenantID, collectionID, ts) - - var r0 []*dbmodel.Field - if rf, ok := ret.Get(0).(func(string, int64, uint64) []*dbmodel.Field); ok { - r0 = rf(tenantID, collectionID, ts) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*dbmodel.Field) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, int64, uint64) error); ok { - r1 = rf(tenantID, collectionID, ts) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Insert provides a mock function with given fields: in -func (_m *IFieldDb) Insert(in []*dbmodel.Field) error { - ret := _m.Called(in) - - var r0 error - if rf, ok := ret.Get(0).(func([]*dbmodel.Field) error); ok { - r0 = rf(in) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -type mockConstructorTestingTNewIFieldDb interface { - mock.TestingT - Cleanup(func()) -} - -// NewIFieldDb creates a new instance of IFieldDb. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewIFieldDb(t mockConstructorTestingTNewIFieldDb) *IFieldDb { - mock := &IFieldDb{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/internal/metastore/db/dbmodel/mocks/IGrantDb.go b/internal/metastore/db/dbmodel/mocks/IGrantDb.go deleted file mode 100644 index 672d9895cf..0000000000 --- a/internal/metastore/db/dbmodel/mocks/IGrantDb.go +++ /dev/null @@ -1,79 +0,0 @@ -// Code generated by mockery v2.14.0. DO NOT EDIT. - -package mocks - -import ( - dbmodel "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - mock "github.com/stretchr/testify/mock" -) - -// IGrantDb is an autogenerated mock type for the IGrantDb type -type IGrantDb struct { - mock.Mock -} - -// Delete provides a mock function with given fields: tenantID, roleID, object, objectName -func (_m *IGrantDb) Delete(tenantID string, roleID int64, object string, objectName string) error { - ret := _m.Called(tenantID, roleID, object, objectName) - - var r0 error - if rf, ok := ret.Get(0).(func(string, int64, string, string) error); ok { - r0 = rf(tenantID, roleID, object, objectName) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// GetGrants provides a mock function with given fields: tenantID, roleID, object, objectName -func (_m *IGrantDb) GetGrants(tenantID string, roleID int64, object string, objectName string) ([]*dbmodel.Grant, error) { - ret := _m.Called(tenantID, roleID, object, objectName) - - var r0 []*dbmodel.Grant - if rf, ok := ret.Get(0).(func(string, int64, string, string) []*dbmodel.Grant); ok { - r0 = rf(tenantID, roleID, object, objectName) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*dbmodel.Grant) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, int64, string, string) error); ok { - r1 = rf(tenantID, roleID, object, objectName) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Insert provides a mock function with given fields: in -func (_m *IGrantDb) Insert(in *dbmodel.Grant) error { - ret := _m.Called(in) - - var r0 error - if rf, ok := ret.Get(0).(func(*dbmodel.Grant) error); ok { - r0 = rf(in) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -type mockConstructorTestingTNewIGrantDb interface { - mock.TestingT - Cleanup(func()) -} - -// NewIGrantDb creates a new instance of IGrantDb. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewIGrantDb(t mockConstructorTestingTNewIGrantDb) *IGrantDb { - mock := &IGrantDb{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/internal/metastore/db/dbmodel/mocks/IGrantIDDb.go b/internal/metastore/db/dbmodel/mocks/IGrantIDDb.go deleted file mode 100644 index 8411ebb982..0000000000 --- a/internal/metastore/db/dbmodel/mocks/IGrantIDDb.go +++ /dev/null @@ -1,79 +0,0 @@ -// Code generated by mockery v2.14.0. DO NOT EDIT. - -package mocks - -import ( - dbmodel "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - mock "github.com/stretchr/testify/mock" -) - -// IGrantIDDb is an autogenerated mock type for the IGrantIDDb type -type IGrantIDDb struct { - mock.Mock -} - -// Delete provides a mock function with given fields: tenantID, grantID, privilege -func (_m *IGrantIDDb) Delete(tenantID string, grantID int64, privilege string) error { - ret := _m.Called(tenantID, grantID, privilege) - - var r0 error - if rf, ok := ret.Get(0).(func(string, int64, string) error); ok { - r0 = rf(tenantID, grantID, privilege) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// GetGrantIDs provides a mock function with given fields: tenantID, grantID, privilege, preloadGrant, preloadGrantor -func (_m *IGrantIDDb) GetGrantIDs(tenantID string, grantID int64, privilege string, preloadGrant bool, preloadGrantor bool) ([]*dbmodel.GrantID, error) { - ret := _m.Called(tenantID, grantID, privilege, preloadGrant, preloadGrantor) - - var r0 []*dbmodel.GrantID - if rf, ok := ret.Get(0).(func(string, int64, string, bool, bool) []*dbmodel.GrantID); ok { - r0 = rf(tenantID, grantID, privilege, preloadGrant, preloadGrantor) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*dbmodel.GrantID) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, int64, string, bool, bool) error); ok { - r1 = rf(tenantID, grantID, privilege, preloadGrant, preloadGrantor) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Insert provides a mock function with given fields: in -func (_m *IGrantIDDb) Insert(in *dbmodel.GrantID) error { - ret := _m.Called(in) - - var r0 error - if rf, ok := ret.Get(0).(func(*dbmodel.GrantID) error); ok { - r0 = rf(in) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -type mockConstructorTestingTNewIGrantIDDb interface { - mock.TestingT - Cleanup(func()) -} - -// NewIGrantIDDb creates a new instance of IGrantIDDb. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewIGrantIDDb(t mockConstructorTestingTNewIGrantIDDb) *IGrantIDDb { - mock := &IGrantIDDb{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/internal/metastore/db/dbmodel/mocks/IIndexDb.go b/internal/metastore/db/dbmodel/mocks/IIndexDb.go deleted file mode 100644 index ee380da921..0000000000 --- a/internal/metastore/db/dbmodel/mocks/IIndexDb.go +++ /dev/null @@ -1,130 +0,0 @@ -// Code generated by mockery v2.14.0. DO NOT EDIT. - -package mocks - -import ( - dbmodel "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - mock "github.com/stretchr/testify/mock" -) - -// IIndexDb is an autogenerated mock type for the IIndexDb type -type IIndexDb struct { - mock.Mock -} - -// Get provides a mock function with given fields: tenantID, collectionID -func (_m *IIndexDb) Get(tenantID string, collectionID int64) ([]*dbmodel.Index, error) { - ret := _m.Called(tenantID, collectionID) - - var r0 []*dbmodel.Index - if rf, ok := ret.Get(0).(func(string, int64) []*dbmodel.Index); ok { - r0 = rf(tenantID, collectionID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*dbmodel.Index) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, int64) error); ok { - r1 = rf(tenantID, collectionID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Insert provides a mock function with given fields: in -func (_m *IIndexDb) Insert(in []*dbmodel.Index) error { - ret := _m.Called(in) - - var r0 error - if rf, ok := ret.Get(0).(func([]*dbmodel.Index) error); ok { - r0 = rf(in) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// List provides a mock function with given fields: tenantID -func (_m *IIndexDb) List(tenantID string) ([]*dbmodel.IndexResult, error) { - ret := _m.Called(tenantID) - - var r0 []*dbmodel.IndexResult - if rf, ok := ret.Get(0).(func(string) []*dbmodel.IndexResult); ok { - r0 = rf(tenantID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*dbmodel.IndexResult) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(tenantID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// MarkDeletedByCollectionID provides a mock function with given fields: tenantID, collID -func (_m *IIndexDb) MarkDeletedByCollectionID(tenantID string, collID int64) error { - ret := _m.Called(tenantID, collID) - - var r0 error - if rf, ok := ret.Get(0).(func(string, int64) error); ok { - r0 = rf(tenantID, collID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// MarkDeletedByIndexID provides a mock function with given fields: tenantID, idxID -func (_m *IIndexDb) MarkDeletedByIndexID(tenantID string, idxID int64) error { - ret := _m.Called(tenantID, idxID) - - var r0 error - if rf, ok := ret.Get(0).(func(string, int64) error); ok { - r0 = rf(tenantID, idxID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Update provides a mock function with given fields: in -func (_m *IIndexDb) Update(in *dbmodel.Index) error { - ret := _m.Called(in) - - var r0 error - if rf, ok := ret.Get(0).(func(*dbmodel.Index) error); ok { - r0 = rf(in) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -type mockConstructorTestingTNewIIndexDb interface { - mock.TestingT - Cleanup(func()) -} - -// NewIIndexDb creates a new instance of IIndexDb. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewIIndexDb(t mockConstructorTestingTNewIIndexDb) *IIndexDb { - mock := &IIndexDb{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/internal/metastore/db/dbmodel/mocks/IMetaDomain.go b/internal/metastore/db/dbmodel/mocks/IMetaDomain.go deleted file mode 100644 index 12a3653cfd..0000000000 --- a/internal/metastore/db/dbmodel/mocks/IMetaDomain.go +++ /dev/null @@ -1,222 +0,0 @@ -// Code generated by mockery v2.14.0. DO NOT EDIT. - -package mocks - -import ( - context "context" - - dbmodel "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - mock "github.com/stretchr/testify/mock" -) - -// IMetaDomain is an autogenerated mock type for the IMetaDomain type -type IMetaDomain struct { - mock.Mock -} - -// CollAliasDb provides a mock function with given fields: ctx -func (_m *IMetaDomain) CollAliasDb(ctx context.Context) dbmodel.ICollAliasDb { - ret := _m.Called(ctx) - - var r0 dbmodel.ICollAliasDb - if rf, ok := ret.Get(0).(func(context.Context) dbmodel.ICollAliasDb); ok { - r0 = rf(ctx) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(dbmodel.ICollAliasDb) - } - } - - return r0 -} - -// CollChannelDb provides a mock function with given fields: ctx -func (_m *IMetaDomain) CollChannelDb(ctx context.Context) dbmodel.ICollChannelDb { - ret := _m.Called(ctx) - - var r0 dbmodel.ICollChannelDb - if rf, ok := ret.Get(0).(func(context.Context) dbmodel.ICollChannelDb); ok { - r0 = rf(ctx) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(dbmodel.ICollChannelDb) - } - } - - return r0 -} - -// CollectionDb provides a mock function with given fields: ctx -func (_m *IMetaDomain) CollectionDb(ctx context.Context) dbmodel.ICollectionDb { - ret := _m.Called(ctx) - - var r0 dbmodel.ICollectionDb - if rf, ok := ret.Get(0).(func(context.Context) dbmodel.ICollectionDb); ok { - r0 = rf(ctx) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(dbmodel.ICollectionDb) - } - } - - return r0 -} - -// FieldDb provides a mock function with given fields: ctx -func (_m *IMetaDomain) FieldDb(ctx context.Context) dbmodel.IFieldDb { - ret := _m.Called(ctx) - - var r0 dbmodel.IFieldDb - if rf, ok := ret.Get(0).(func(context.Context) dbmodel.IFieldDb); ok { - r0 = rf(ctx) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(dbmodel.IFieldDb) - } - } - - return r0 -} - -// GrantDb provides a mock function with given fields: ctx -func (_m *IMetaDomain) GrantDb(ctx context.Context) dbmodel.IGrantDb { - ret := _m.Called(ctx) - - var r0 dbmodel.IGrantDb - if rf, ok := ret.Get(0).(func(context.Context) dbmodel.IGrantDb); ok { - r0 = rf(ctx) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(dbmodel.IGrantDb) - } - } - - return r0 -} - -// GrantIDDb provides a mock function with given fields: ctx -func (_m *IMetaDomain) GrantIDDb(ctx context.Context) dbmodel.IGrantIDDb { - ret := _m.Called(ctx) - - var r0 dbmodel.IGrantIDDb - if rf, ok := ret.Get(0).(func(context.Context) dbmodel.IGrantIDDb); ok { - r0 = rf(ctx) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(dbmodel.IGrantIDDb) - } - } - - return r0 -} - -// IndexDb provides a mock function with given fields: ctx -func (_m *IMetaDomain) IndexDb(ctx context.Context) dbmodel.IIndexDb { - ret := _m.Called(ctx) - - var r0 dbmodel.IIndexDb - if rf, ok := ret.Get(0).(func(context.Context) dbmodel.IIndexDb); ok { - r0 = rf(ctx) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(dbmodel.IIndexDb) - } - } - - return r0 -} - -// PartitionDb provides a mock function with given fields: ctx -func (_m *IMetaDomain) PartitionDb(ctx context.Context) dbmodel.IPartitionDb { - ret := _m.Called(ctx) - - var r0 dbmodel.IPartitionDb - if rf, ok := ret.Get(0).(func(context.Context) dbmodel.IPartitionDb); ok { - r0 = rf(ctx) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(dbmodel.IPartitionDb) - } - } - - return r0 -} - -// RoleDb provides a mock function with given fields: ctx -func (_m *IMetaDomain) RoleDb(ctx context.Context) dbmodel.IRoleDb { - ret := _m.Called(ctx) - - var r0 dbmodel.IRoleDb - if rf, ok := ret.Get(0).(func(context.Context) dbmodel.IRoleDb); ok { - r0 = rf(ctx) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(dbmodel.IRoleDb) - } - } - - return r0 -} - -// SegmentIndexDb provides a mock function with given fields: ctx -func (_m *IMetaDomain) SegmentIndexDb(ctx context.Context) dbmodel.ISegmentIndexDb { - ret := _m.Called(ctx) - - var r0 dbmodel.ISegmentIndexDb - if rf, ok := ret.Get(0).(func(context.Context) dbmodel.ISegmentIndexDb); ok { - r0 = rf(ctx) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(dbmodel.ISegmentIndexDb) - } - } - - return r0 -} - -// UserDb provides a mock function with given fields: ctx -func (_m *IMetaDomain) UserDb(ctx context.Context) dbmodel.IUserDb { - ret := _m.Called(ctx) - - var r0 dbmodel.IUserDb - if rf, ok := ret.Get(0).(func(context.Context) dbmodel.IUserDb); ok { - r0 = rf(ctx) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(dbmodel.IUserDb) - } - } - - return r0 -} - -// UserRoleDb provides a mock function with given fields: ctx -func (_m *IMetaDomain) UserRoleDb(ctx context.Context) dbmodel.IUserRoleDb { - ret := _m.Called(ctx) - - var r0 dbmodel.IUserRoleDb - if rf, ok := ret.Get(0).(func(context.Context) dbmodel.IUserRoleDb); ok { - r0 = rf(ctx) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(dbmodel.IUserRoleDb) - } - } - - return r0 -} - -type mockConstructorTestingTNewIMetaDomain interface { - mock.TestingT - Cleanup(func()) -} - -// NewIMetaDomain creates a new instance of IMetaDomain. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewIMetaDomain(t mockConstructorTestingTNewIMetaDomain) *IMetaDomain { - mock := &IMetaDomain{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/internal/metastore/db/dbmodel/mocks/IPartitionDb.go b/internal/metastore/db/dbmodel/mocks/IPartitionDb.go deleted file mode 100644 index 194341b218..0000000000 --- a/internal/metastore/db/dbmodel/mocks/IPartitionDb.go +++ /dev/null @@ -1,79 +0,0 @@ -// Code generated by mockery v2.14.0. DO NOT EDIT. - -package mocks - -import ( - dbmodel "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - mock "github.com/stretchr/testify/mock" -) - -// IPartitionDb is an autogenerated mock type for the IPartitionDb type -type IPartitionDb struct { - mock.Mock -} - -// GetByCollectionID provides a mock function with given fields: tenantID, collectionID, ts -func (_m *IPartitionDb) GetByCollectionID(tenantID string, collectionID int64, ts uint64) ([]*dbmodel.Partition, error) { - ret := _m.Called(tenantID, collectionID, ts) - - var r0 []*dbmodel.Partition - if rf, ok := ret.Get(0).(func(string, int64, uint64) []*dbmodel.Partition); ok { - r0 = rf(tenantID, collectionID, ts) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*dbmodel.Partition) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, int64, uint64) error); ok { - r1 = rf(tenantID, collectionID, ts) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Insert provides a mock function with given fields: in -func (_m *IPartitionDb) Insert(in []*dbmodel.Partition) error { - ret := _m.Called(in) - - var r0 error - if rf, ok := ret.Get(0).(func([]*dbmodel.Partition) error); ok { - r0 = rf(in) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Update provides a mock function with given fields: in -func (_m *IPartitionDb) Update(in *dbmodel.Partition) error { - ret := _m.Called(in) - - var r0 error - if rf, ok := ret.Get(0).(func(*dbmodel.Partition) error); ok { - r0 = rf(in) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -type mockConstructorTestingTNewIPartitionDb interface { - mock.TestingT - Cleanup(func()) -} - -// NewIPartitionDb creates a new instance of IPartitionDb. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewIPartitionDb(t mockConstructorTestingTNewIPartitionDb) *IPartitionDb { - mock := &IPartitionDb{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/internal/metastore/db/dbmodel/mocks/IRoleDb.go b/internal/metastore/db/dbmodel/mocks/IRoleDb.go deleted file mode 100644 index 8e441154aa..0000000000 --- a/internal/metastore/db/dbmodel/mocks/IRoleDb.go +++ /dev/null @@ -1,79 +0,0 @@ -// Code generated by mockery v2.14.0. DO NOT EDIT. - -package mocks - -import ( - dbmodel "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - mock "github.com/stretchr/testify/mock" -) - -// IRoleDb is an autogenerated mock type for the IRoleDb type -type IRoleDb struct { - mock.Mock -} - -// Delete provides a mock function with given fields: tenantID, name -func (_m *IRoleDb) Delete(tenantID string, name string) error { - ret := _m.Called(tenantID, name) - - var r0 error - if rf, ok := ret.Get(0).(func(string, string) error); ok { - r0 = rf(tenantID, name) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// GetRoles provides a mock function with given fields: tenantID, name -func (_m *IRoleDb) GetRoles(tenantID string, name string) ([]*dbmodel.Role, error) { - ret := _m.Called(tenantID, name) - - var r0 []*dbmodel.Role - if rf, ok := ret.Get(0).(func(string, string) []*dbmodel.Role); ok { - r0 = rf(tenantID, name) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*dbmodel.Role) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, string) error); ok { - r1 = rf(tenantID, name) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Insert provides a mock function with given fields: in -func (_m *IRoleDb) Insert(in *dbmodel.Role) error { - ret := _m.Called(in) - - var r0 error - if rf, ok := ret.Get(0).(func(*dbmodel.Role) error); ok { - r0 = rf(in) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -type mockConstructorTestingTNewIRoleDb interface { - mock.TestingT - Cleanup(func()) -} - -// NewIRoleDb creates a new instance of IRoleDb. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewIRoleDb(t mockConstructorTestingTNewIRoleDb) *IRoleDb { - mock := &IRoleDb{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/internal/metastore/db/dbmodel/mocks/ISegmentIndexDb.go b/internal/metastore/db/dbmodel/mocks/ISegmentIndexDb.go deleted file mode 100644 index 09827620f0..0000000000 --- a/internal/metastore/db/dbmodel/mocks/ISegmentIndexDb.go +++ /dev/null @@ -1,144 +0,0 @@ -// Code generated by mockery v2.14.0. DO NOT EDIT. - -package mocks - -import ( - dbmodel "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - mock "github.com/stretchr/testify/mock" -) - -// ISegmentIndexDb is an autogenerated mock type for the ISegmentIndexDb type -type ISegmentIndexDb struct { - mock.Mock -} - -// Get provides a mock function with given fields: tenantID, collectionID, buildID -func (_m *ISegmentIndexDb) Get(tenantID string, collectionID int64, buildID int64) ([]*dbmodel.SegmentIndexResult, error) { - ret := _m.Called(tenantID, collectionID, buildID) - - var r0 []*dbmodel.SegmentIndexResult - if rf, ok := ret.Get(0).(func(string, int64, int64) []*dbmodel.SegmentIndexResult); ok { - r0 = rf(tenantID, collectionID, buildID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*dbmodel.SegmentIndexResult) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, int64, int64) error); ok { - r1 = rf(tenantID, collectionID, buildID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Insert provides a mock function with given fields: in -func (_m *ISegmentIndexDb) Insert(in []*dbmodel.SegmentIndex) error { - ret := _m.Called(in) - - var r0 error - if rf, ok := ret.Get(0).(func([]*dbmodel.SegmentIndex) error); ok { - r0 = rf(in) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// List provides a mock function with given fields: tenantID -func (_m *ISegmentIndexDb) List(tenantID string) ([]*dbmodel.SegmentIndexResult, error) { - ret := _m.Called(tenantID) - - var r0 []*dbmodel.SegmentIndexResult - if rf, ok := ret.Get(0).(func(string) []*dbmodel.SegmentIndexResult); ok { - r0 = rf(tenantID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*dbmodel.SegmentIndexResult) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(tenantID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// MarkDeleted provides a mock function with given fields: tenantID, in -func (_m *ISegmentIndexDb) MarkDeleted(tenantID string, in []*dbmodel.SegmentIndex) error { - ret := _m.Called(tenantID, in) - - var r0 error - if rf, ok := ret.Get(0).(func(string, []*dbmodel.SegmentIndex) error); ok { - r0 = rf(tenantID, in) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// MarkDeletedByBuildID provides a mock function with given fields: tenantID, idxID -func (_m *ISegmentIndexDb) MarkDeletedByBuildID(tenantID string, idxID int64) error { - ret := _m.Called(tenantID, idxID) - - var r0 error - if rf, ok := ret.Get(0).(func(string, int64) error); ok { - r0 = rf(tenantID, idxID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// MarkDeletedByCollectionID provides a mock function with given fields: tenantID, collID -func (_m *ISegmentIndexDb) MarkDeletedByCollectionID(tenantID string, collID int64) error { - ret := _m.Called(tenantID, collID) - - var r0 error - if rf, ok := ret.Get(0).(func(string, int64) error); ok { - r0 = rf(tenantID, collID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// Update provides a mock function with given fields: in -func (_m *ISegmentIndexDb) Update(in *dbmodel.SegmentIndex) error { - ret := _m.Called(in) - - var r0 error - if rf, ok := ret.Get(0).(func(*dbmodel.SegmentIndex) error); ok { - r0 = rf(in) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -type mockConstructorTestingTNewISegmentIndexDb interface { - mock.TestingT - Cleanup(func()) -} - -// NewISegmentIndexDb creates a new instance of ISegmentIndexDb. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewISegmentIndexDb(t mockConstructorTestingTNewISegmentIndexDb) *ISegmentIndexDb { - mock := &ISegmentIndexDb{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/internal/metastore/db/dbmodel/mocks/IUserDb.go b/internal/metastore/db/dbmodel/mocks/IUserDb.go deleted file mode 100644 index 8ab147a705..0000000000 --- a/internal/metastore/db/dbmodel/mocks/IUserDb.go +++ /dev/null @@ -1,116 +0,0 @@ -// Code generated by mockery v2.14.0. DO NOT EDIT. - -package mocks - -import ( - dbmodel "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - mock "github.com/stretchr/testify/mock" -) - -// IUserDb is an autogenerated mock type for the IUserDb type -type IUserDb struct { - mock.Mock -} - -// GetByUsername provides a mock function with given fields: tenantID, username -func (_m *IUserDb) GetByUsername(tenantID string, username string) (*dbmodel.User, error) { - ret := _m.Called(tenantID, username) - - var r0 *dbmodel.User - if rf, ok := ret.Get(0).(func(string, string) *dbmodel.User); ok { - r0 = rf(tenantID, username) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).(*dbmodel.User) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, string) error); ok { - r1 = rf(tenantID, username) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Insert provides a mock function with given fields: in -func (_m *IUserDb) Insert(in *dbmodel.User) error { - ret := _m.Called(in) - - var r0 error - if rf, ok := ret.Get(0).(func(*dbmodel.User) error); ok { - r0 = rf(in) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// ListUser provides a mock function with given fields: tenantID -func (_m *IUserDb) ListUser(tenantID string) ([]*dbmodel.User, error) { - ret := _m.Called(tenantID) - - var r0 []*dbmodel.User - if rf, ok := ret.Get(0).(func(string) []*dbmodel.User); ok { - r0 = rf(tenantID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*dbmodel.User) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string) error); ok { - r1 = rf(tenantID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// MarkDeletedByUsername provides a mock function with given fields: tenantID, username -func (_m *IUserDb) MarkDeletedByUsername(tenantID string, username string) error { - ret := _m.Called(tenantID, username) - - var r0 error - if rf, ok := ret.Get(0).(func(string, string) error); ok { - r0 = rf(tenantID, username) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// UpdatePassword provides a mock function with given fields: tenantID, username, encryptedPassword -func (_m *IUserDb) UpdatePassword(tenantID string, username string, encryptedPassword string) error { - ret := _m.Called(tenantID, username, encryptedPassword) - - var r0 error - if rf, ok := ret.Get(0).(func(string, string, string) error); ok { - r0 = rf(tenantID, username, encryptedPassword) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -type mockConstructorTestingTNewIUserDb interface { - mock.TestingT - Cleanup(func()) -} - -// NewIUserDb creates a new instance of IUserDb. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewIUserDb(t mockConstructorTestingTNewIUserDb) *IUserDb { - mock := &IUserDb{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/internal/metastore/db/dbmodel/mocks/IUserRoleDb.go b/internal/metastore/db/dbmodel/mocks/IUserRoleDb.go deleted file mode 100644 index 2612848ee7..0000000000 --- a/internal/metastore/db/dbmodel/mocks/IUserRoleDb.go +++ /dev/null @@ -1,79 +0,0 @@ -// Code generated by mockery v2.14.0. DO NOT EDIT. - -package mocks - -import ( - dbmodel "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - mock "github.com/stretchr/testify/mock" -) - -// IUserRoleDb is an autogenerated mock type for the IUserRoleDb type -type IUserRoleDb struct { - mock.Mock -} - -// Delete provides a mock function with given fields: tenantID, userID, roleID -func (_m *IUserRoleDb) Delete(tenantID string, userID int64, roleID int64) error { - ret := _m.Called(tenantID, userID, roleID) - - var r0 error - if rf, ok := ret.Get(0).(func(string, int64, int64) error); ok { - r0 = rf(tenantID, userID, roleID) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -// GetUserRoles provides a mock function with given fields: tenantID, userID, roleID -func (_m *IUserRoleDb) GetUserRoles(tenantID string, userID int64, roleID int64) ([]*dbmodel.UserRole, error) { - ret := _m.Called(tenantID, userID, roleID) - - var r0 []*dbmodel.UserRole - if rf, ok := ret.Get(0).(func(string, int64, int64) []*dbmodel.UserRole); ok { - r0 = rf(tenantID, userID, roleID) - } else { - if ret.Get(0) != nil { - r0 = ret.Get(0).([]*dbmodel.UserRole) - } - } - - var r1 error - if rf, ok := ret.Get(1).(func(string, int64, int64) error); ok { - r1 = rf(tenantID, userID, roleID) - } else { - r1 = ret.Error(1) - } - - return r0, r1 -} - -// Insert provides a mock function with given fields: in -func (_m *IUserRoleDb) Insert(in *dbmodel.UserRole) error { - ret := _m.Called(in) - - var r0 error - if rf, ok := ret.Get(0).(func(*dbmodel.UserRole) error); ok { - r0 = rf(in) - } else { - r0 = ret.Error(0) - } - - return r0 -} - -type mockConstructorTestingTNewIUserRoleDb interface { - mock.TestingT - Cleanup(func()) -} - -// NewIUserRoleDb creates a new instance of IUserRoleDb. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -func NewIUserRoleDb(t mockConstructorTestingTNewIUserRoleDb) *IUserRoleDb { - mock := &IUserRoleDb{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/internal/metastore/db/dbmodel/partition.go b/internal/metastore/db/dbmodel/partition.go deleted file mode 100644 index d20f1cee2a..0000000000 --- a/internal/metastore/db/dbmodel/partition.go +++ /dev/null @@ -1,54 +0,0 @@ -package dbmodel - -import ( - "time" - - "github.com/milvus-io/milvus/internal/metastore/model" - "github.com/milvus-io/milvus/pkg/util/typeutil" -) - -type Partition struct { - ID int64 `gorm:"id"` - TenantID string `gorm:"tenant_id"` - PartitionID int64 `gorm:"partition_id"` - PartitionName string `gorm:"partition_name"` - PartitionCreatedTimestamp uint64 `gorm:"partition_created_timestamp"` - CollectionID int64 `gorm:"collection_id"` - Status int32 `gorm:"status"` - Ts typeutil.Timestamp `gorm:"ts"` - IsDeleted bool `gorm:"is_deleted"` - CreatedAt time.Time `gorm:"created_at"` - UpdatedAt time.Time `gorm:"updated_at"` -} - -func (v Partition) TableName() string { - return "partitions" -} - -//go:generate mockery --name=IPartitionDb -type IPartitionDb interface { - GetByCollectionID(tenantID string, collectionID typeutil.UniqueID, ts typeutil.Timestamp) ([]*Partition, error) - Insert(in []*Partition) error - Update(in *Partition) error - //MarkDeleted(tenantID string, collID typeutil.UniqueID) error -} - -// model <---> db - -func UnmarshalPartitionModel(partitons []*Partition) []*model.Partition { - r := make([]*model.Partition, 0, len(partitons)) - for _, p := range partitons { - partition := ConvertPartitionDBToModel(p) - r = append(r, partition) - } - - return r -} - -func ConvertPartitionDBToModel(partiton *Partition) *model.Partition { - return &model.Partition{ - PartitionID: partiton.PartitionID, - PartitionName: partiton.PartitionName, - PartitionCreatedTimestamp: partiton.PartitionCreatedTimestamp, - } -} diff --git a/internal/metastore/db/dbmodel/role.go b/internal/metastore/db/dbmodel/role.go deleted file mode 100644 index ca6a6d2bb0..0000000000 --- a/internal/metastore/db/dbmodel/role.go +++ /dev/null @@ -1,23 +0,0 @@ -package dbmodel - -import "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" - -type Role struct { - Base - Name string `gorm:"name"` -} - -func (r *Role) TableName() string { - return "role" -} - -func (r *Role) Unmarshal() *milvuspb.RoleEntity { - return &milvuspb.RoleEntity{Name: r.Name} -} - -//go:generate mockery --name=IRoleDb -type IRoleDb interface { - GetRoles(tenantID string, name string) ([]*Role, error) - Insert(in *Role) error - Delete(tenantID string, name string) error -} diff --git a/internal/metastore/db/dbmodel/segment_index.go b/internal/metastore/db/dbmodel/segment_index.go deleted file mode 100644 index b4738c11de..0000000000 --- a/internal/metastore/db/dbmodel/segment_index.go +++ /dev/null @@ -1,99 +0,0 @@ -package dbmodel - -import ( - "time" - - "github.com/milvus-io/milvus/pkg/util/typeutil" -) - -type SegmentIndex struct { - ID int64 `gorm:"id"` - TenantID string `gorm:"tenant_id"` - // SegmentIndexInfo (CollectionID & PartitionID & SegmentID & FieldID & IndexID & BuildID & EnableIndex) - CollectionID int64 `gorm:"collection_id"` - PartitionID int64 `gorm:"partition_id"` - SegmentID int64 `gorm:"segment_id"` - NumRows int64 `gorm:"num_rows"` - // IndexInfo (IndexID & IndexName & IndexParams) - IndexID int64 `gorm:"index_id"` - BuildID int64 `gorm:"build_id"` - NodeID int64 `gorm:"node_id"` - IndexVersion int64 `gorm:"index_version"` - IndexState int32 `gorm:"index_state"` - FailReason string `gorm:"fail_reason"` - CreateTime uint64 `gorm:"create_time"` - IndexFileKeys string `gorm:"index_file_keys"` - IndexSize uint64 `gorm:"index_size"` - IsDeleted bool `gorm:"is_deleted"` - CreatedAt time.Time `gorm:"created_at"` - UpdatedAt time.Time `gorm:"updated_at"` -} - -func (v SegmentIndex) TableName() string { - return "segment_indexes" -} - -type SegmentIndexResult struct { - CollectionID int64 - PartitionID int64 - SegmentID int64 - NumRows int64 - IndexID int64 - BuildID int64 - NodeID int64 - IndexVersion int64 - IndexState int32 - FailReason string - IsDeleted bool - CreateTime uint64 - IndexFileKeys string - IndexSize uint64 -} - -//go:generate mockery --name=ISegmentIndexDb -type ISegmentIndexDb interface { - Get(tenantID string, collectionID, buildID typeutil.UniqueID) ([]*SegmentIndexResult, error) - List(tenantID string) ([]*SegmentIndexResult, error) - Insert(in []*SegmentIndex) error - Update(in *SegmentIndex) error - MarkDeleted(tenantID string, in []*SegmentIndex) error - MarkDeletedByCollectionID(tenantID string, collID typeutil.UniqueID) error - MarkDeletedByBuildID(tenantID string, idxID typeutil.UniqueID) error -} - -//func UnmarshalSegmentIndexModel(inputs []*SegmentIndexResult) ([]*model.SegmentIndex, error) { -// result := make([]*model.SegmentIndex, 0, len(inputs)) -// for _, ir := range inputs { -// -// var IndexFileKeys []string -// if ir.IndexFileKeys != "" { -// err := json.Unmarshal([]byte(ir.IndexFileKeys), &IndexFileKeys) -// if err != nil { -// log.Error("unmarshal index file paths of segment index failed", zap.Int64("collectionID", ir.CollectionID), -// zap.Int64("indexID", ir.IndexID), zap.Int64("segmentID", ir.SegmentID), -// zap.Int64("buildID", ir.BuildID), zap.Error(err)) -// return nil, err -// } -// } -// -// idx := &model.SegmentIndex{ -// SegmentID: ir.SegmentID, -// CollectionID: ir.CollectionID, -// PartitionID: ir.PartitionID, -// NumRows: ir.NumRows, -// IndexID: ir.IndexID, -// BuildID: ir.BuildID, -// NodeID: ir.NodeID, -// IndexVersion: ir.IndexVersion, -// IndexState: commonpb.IndexState(ir.IndexState), -// FailReason: ir.FailReason, -// IsDeleted: ir.IsDeleted, -// CreateTime: ir.CreateTime, -// IndexFileKeys: IndexFileKeys, -// IndexSize: ir.IndexSize, -// } -// result = append(result, idx) -// } -// -// return result, nil -//} diff --git a/internal/metastore/db/dbmodel/user.go b/internal/metastore/db/dbmodel/user.go deleted file mode 100644 index a4a25c8b12..0000000000 --- a/internal/metastore/db/dbmodel/user.go +++ /dev/null @@ -1,44 +0,0 @@ -package dbmodel - -import ( - "time" - - "github.com/milvus-io/milvus/internal/metastore/model" -) - -type User struct { - ID int64 `gorm:"id"` - TenantID string `gorm:"tenant_id"` - Username string `gorm:"username"` - EncryptedPassword string `gorm:"encrypted_password"` - IsSuper bool `gorm:"is_super"` - IsDeleted bool `gorm:"is_deleted"` - CreatedAt time.Time `gorm:"created_at"` - UpdatedAt time.Time `gorm:"updated_at"` -} - -func (v User) TableName() string { - return "credential_users" -} - -//go:generate mockery --name=IUserDb -type IUserDb interface { - GetByUsername(tenantID string, username string) (*User, error) - ListUser(tenantID string) ([]*User, error) - Insert(in *User) error - MarkDeletedByUsername(tenantID string, username string) error - UpdatePassword(tenantID string, username string, encryptedPassword string) error -} - -// model <---> db - -func UnmarshalUserModel(user *User) *model.Credential { - if user == nil { - return nil - } - - return &model.Credential{ - Username: user.Username, - EncryptedPassword: user.EncryptedPassword, - } -} diff --git a/internal/metastore/db/dbmodel/user_role.go b/internal/metastore/db/dbmodel/user_role.go deleted file mode 100644 index f8d536448a..0000000000 --- a/internal/metastore/db/dbmodel/user_role.go +++ /dev/null @@ -1,21 +0,0 @@ -package dbmodel - -type UserRole struct { - Base - UserID int64 `gorm:"user_id"` - RoleID int64 `gorm:"role_id"` - - User User `gorm:"foreignKey:UserID"` - Role Role `gorm:"foreignKey:RoleID"` -} - -func (u *UserRole) TableName() string { - return "user_role" -} - -//go:generate mockery --name=IUserRoleDb -type IUserRoleDb interface { - GetUserRoles(tenantID string, userID int64, roleID int64) ([]*UserRole, error) - Insert(in *UserRole) error - Delete(tenantID string, userID int64, roleID int64) error -} diff --git a/internal/metastore/db/rootcoord/table_catalog.go b/internal/metastore/db/rootcoord/table_catalog.go deleted file mode 100644 index 3d1edca074..0000000000 --- a/internal/metastore/db/rootcoord/table_catalog.go +++ /dev/null @@ -1,1003 +0,0 @@ -package rootcoord - -import ( - "context" - "encoding/json" - "fmt" - "runtime" - "time" - - "github.com/milvus-io/milvus/pkg/util/tsoutil" - "go.uber.org/zap" - - "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" - "github.com/milvus-io/milvus/internal/metastore" - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - "github.com/milvus-io/milvus/internal/metastore/model" - "github.com/milvus-io/milvus/pkg/common" - "github.com/milvus-io/milvus/pkg/log" - "github.com/milvus-io/milvus/pkg/util" - "github.com/milvus-io/milvus/pkg/util/contextutil" - "github.com/milvus-io/milvus/pkg/util/funcutil" - "github.com/milvus-io/milvus/pkg/util/typeutil" -) - -type Catalog struct { - metaDomain dbmodel.IMetaDomain - txImpl dbmodel.ITransaction -} - -func NewTableCatalog(txImpl dbmodel.ITransaction, metaDomain dbmodel.IMetaDomain) *Catalog { - return &Catalog{ - txImpl: txImpl, - metaDomain: metaDomain, - } -} - -func (tc *Catalog) CreateDatabase(ctx context.Context, db *model.Database, ts typeutil.Timestamp) error { - //TODO - return nil -} - -func (tc *Catalog) DropDatabase(ctx context.Context, dbID int64, ts typeutil.Timestamp) error { - //TODO - return nil - -} - -func (tc *Catalog) ListDatabases(ctx context.Context, ts typeutil.Timestamp) ([]*model.Database, error) { - //TODO - return make([]*model.Database, 0), nil -} - -func (tc *Catalog) CreateCollection(ctx context.Context, collection *model.Collection, ts typeutil.Timestamp) error { - tenantID := contextutil.TenantID(ctx) - - return tc.txImpl.Transaction(ctx, func(txCtx context.Context) error { - // insert collection - var startPositionsStr string - if collection.StartPositions != nil { - startPositionsBytes, err := json.Marshal(collection.StartPositions) - if err != nil { - log.Error("marshal collection start positions error", zap.Int64("collectionID", collection.CollectionID), zap.Uint64("ts", ts), zap.Error(err)) - return err - } - startPositionsStr = string(startPositionsBytes) - } - - properties, err := dbmodel.MarshalProperties(collection.Properties) - if err != nil { - return err - } - - err = tc.metaDomain.CollectionDb(txCtx).Insert(&dbmodel.Collection{ - TenantID: tenantID, - CollectionID: collection.CollectionID, - CollectionName: collection.Name, - Description: collection.Description, - AutoID: collection.AutoID, - ShardsNum: collection.ShardsNum, - StartPosition: startPositionsStr, - ConsistencyLevel: int32(collection.ConsistencyLevel), - Status: int32(collection.State), - Ts: ts, - Properties: properties, - }) - if err != nil { - return err - } - - // insert field - var fields = make([]*dbmodel.Field, 0, len(collection.Fields)) - for _, field := range collection.Fields { - typeParamsBytes, err := json.Marshal(field.TypeParams) - if err != nil { - log.Error("marshal TypeParams of field failed", zap.Error(err)) - return err - } - typeParamsStr := string(typeParamsBytes) - - indexParamsBytes, err := json.Marshal(field.IndexParams) - if err != nil { - log.Error("marshal IndexParams of field failed", zap.Error(err)) - return err - } - indexParamsStr := string(indexParamsBytes) - - f := &dbmodel.Field{ - TenantID: collection.TenantID, - FieldID: field.FieldID, - FieldName: field.Name, - IsPrimaryKey: field.IsPrimaryKey, - Description: field.Description, - DataType: field.DataType, - TypeParams: typeParamsStr, - IndexParams: indexParamsStr, - AutoID: field.AutoID, - CollectionID: collection.CollectionID, - Ts: ts, - } - - fields = append(fields, f) - } - - err = tc.metaDomain.FieldDb(txCtx).Insert(fields) - if err != nil { - return err - } - - // insert partition - var partitions = make([]*dbmodel.Partition, 0, len(collection.Partitions)) - for _, partition := range collection.Partitions { - p := &dbmodel.Partition{ - TenantID: collection.TenantID, - PartitionID: partition.PartitionID, - PartitionName: partition.PartitionName, - PartitionCreatedTimestamp: partition.PartitionCreatedTimestamp, - CollectionID: collection.CollectionID, - Ts: ts, - } - partitions = append(partitions, p) - } - - err = tc.metaDomain.PartitionDb(txCtx).Insert(partitions) - if err != nil { - return err - } - - // insert channel - var channels = make([]*dbmodel.CollectionChannel, 0, len(collection.VirtualChannelNames)) - for i, vChannelName := range collection.VirtualChannelNames { - collChannel := &dbmodel.CollectionChannel{ - TenantID: collection.TenantID, - CollectionID: collection.CollectionID, - VirtualChannelName: vChannelName, - PhysicalChannelName: collection.PhysicalChannelNames[i], - Ts: ts, - } - channels = append(channels, collChannel) - } - - err = tc.metaDomain.CollChannelDb(txCtx).Insert(channels) - if err != nil { - return err - } - - return nil - }) -} - -func (tc *Catalog) GetCollectionByID(ctx context.Context, dbID int64, ts typeutil.Timestamp, collectionID typeutil.UniqueID) (*model.Collection, error) { - tenantID := contextutil.TenantID(ctx) - - // get latest timestamp less than or equals to param ts - cidTsPair, err := tc.metaDomain.CollectionDb(ctx).GetCollectionIDTs(tenantID, collectionID, ts) - if err != nil { - return nil, err - } - if cidTsPair.IsDeleted { - log.Error("not found collection", zap.Int64("collectionID", collectionID), zap.Uint64("ts", ts)) - return nil, fmt.Errorf("not found collection, collID=%d, ts=%d", collectionID, ts) - } - - queryTs := cidTsPair.Ts - - return tc.populateCollection(ctx, collectionID, queryTs) -} - -func (tc *Catalog) populateCollection(ctx context.Context, collectionID typeutil.UniqueID, ts typeutil.Timestamp) (*model.Collection, error) { - tenantID := contextutil.TenantID(ctx) - - // get collection by collection_id and ts - collection, err := tc.metaDomain.CollectionDb(ctx).Get(tenantID, collectionID, ts) - if err != nil { - return nil, err - } - - // get fields by collection_id and ts - fields, err := tc.metaDomain.FieldDb(ctx).GetByCollectionID(tenantID, collectionID, ts) - if err != nil { - return nil, err - } - - // get partitions by collection_id and ts - partitions, err := tc.metaDomain.PartitionDb(ctx).GetByCollectionID(tenantID, collectionID, ts) - if err != nil { - return nil, err - } - - // get channels by collection_id and ts - channels, err := tc.metaDomain.CollChannelDb(ctx).GetByCollectionID(tenantID, collectionID, ts) - if err != nil { - return nil, err - } - - // merge as collection attributes - - mCollection, err := dbmodel.UnmarshalCollectionModel(collection) - if err != nil { - return nil, err - } - - mFields, err := dbmodel.UnmarshalFieldModel(fields) - if err != nil { - return nil, err - } - - mCollection.Fields = mFields - mCollection.Partitions = dbmodel.UnmarshalPartitionModel(partitions) - mCollection.VirtualChannelNames, mCollection.PhysicalChannelNames = dbmodel.ExtractChannelNames(channels) - - return mCollection, nil -} - -func (tc *Catalog) GetCollectionByName(ctx context.Context, dbID int64, collectionName string, ts typeutil.Timestamp) (*model.Collection, error) { - tenantID := contextutil.TenantID(ctx) - - // Since collection name will not change for different ts - collectionID, err := tc.metaDomain.CollectionDb(ctx).GetCollectionIDByName(tenantID, collectionName, ts) - if err != nil { - return nil, err - } - - return tc.GetCollectionByID(ctx, dbID, ts, collectionID) -} - -// ListCollections For time travel (ts > 0), find only one record respectively for each collection no matter `is_deleted` is true or false -// i.e. there are 3 collections in total, -// [collection1, t1, is_deleted=true] -// [collection2, t2, is_deleted=false] -// [collection3, t3, is_deleted=false] -// t1, t2, t3 are the largest timestamp that less than or equal to @param ts -// the final result will only return collection2 and collection3 since collection1 is deleted -func (tc *Catalog) ListCollections(ctx context.Context, dbID int64, ts typeutil.Timestamp) ([]*model.Collection, error) { - tenantID := contextutil.TenantID(ctx) - - // 1. find each collection_id with latest ts <= @param ts - cidTsPairs, err := tc.metaDomain.CollectionDb(ctx).ListCollectionIDTs(tenantID, ts) - if err != nil { - return nil, err - } - if len(cidTsPairs) == 0 { - return make([]*model.Collection, 0), nil - } - - // 2. populate each collection - collections := make([]*model.Collection, len(cidTsPairs)) - - reloadCollectionByCollectionIDTsFunc := func(idx int) error { - collIDTsPair := cidTsPairs[idx] - collection, err := tc.populateCollection(ctx, collIDTsPair.CollectionID, collIDTsPair.Ts) - if err != nil { - return err - } - collections[idx] = collection - return nil - } - - concurrency := len(cidTsPairs) - if concurrency > runtime.NumCPU() { - concurrency = runtime.NumCPU() - } - err = funcutil.ProcessFuncParallel(len(cidTsPairs), concurrency, reloadCollectionByCollectionIDTsFunc, "ListCollectionByCollectionIDTs") - if err != nil { - log.Error("list collections by collection_id & ts pair failed", zap.Uint64("ts", ts), zap.Error(err)) - return nil, err - } - return collections, nil -} - -func (tc *Catalog) CollectionExists(ctx context.Context, dbID int64, collectionID typeutil.UniqueID, ts typeutil.Timestamp) bool { - tenantID := contextutil.TenantID(ctx) - - // get latest timestamp less than or equals to param ts - cidTsPair, err := tc.metaDomain.CollectionDb(ctx).GetCollectionIDTs(tenantID, collectionID, ts) - if err != nil { - return false - } - if cidTsPair.IsDeleted { - return false - } - - queryTs := cidTsPair.Ts - - col, err := tc.metaDomain.CollectionDb(ctx).Get(tenantID, collectionID, queryTs) - if err != nil { - return false - } - - if col != nil { - return !col.IsDeleted - } - - return false -} - -func (tc *Catalog) DropCollection(ctx context.Context, collection *model.Collection, ts typeutil.Timestamp) error { - tenantID := contextutil.TenantID(ctx) - - return tc.txImpl.Transaction(ctx, func(txCtx context.Context) error { - // 1. insert a mark-deleted record for collections - coll := &dbmodel.Collection{ - TenantID: tenantID, - CollectionID: collection.CollectionID, - Ts: ts, - IsDeleted: true, - } - err := tc.metaDomain.CollectionDb(txCtx).Insert(coll) - if err != nil { - log.Error("insert tombstone record for collections failed", zap.String("tenant", tenantID), zap.Int64("collectionID", collection.CollectionID), zap.Uint64("ts", ts), zap.Error(err)) - return err - } - - // 2. insert a mark-deleted record for collection_aliases - if len(collection.Aliases) > 0 { - collAliases := make([]*dbmodel.CollectionAlias, 0, len(collection.Aliases)) - - for _, alias := range collection.Aliases { - collAliases = append(collAliases, &dbmodel.CollectionAlias{ - TenantID: tenantID, - CollectionID: collection.CollectionID, - CollectionAlias: alias, - Ts: ts, - IsDeleted: true, - }) - } - - err = tc.metaDomain.CollAliasDb(txCtx).Insert(collAliases) - if err != nil { - log.Error("insert tombstone record for collection_aliases failed", zap.String("tenant", tenantID), zap.Int64("collectionID", collection.CollectionID), zap.Uint64("ts", ts), zap.Error(err)) - return err - } - } - - // 3. insert a mark-deleted record for collection_channels - collChannel := &dbmodel.CollectionChannel{ - TenantID: tenantID, - CollectionID: collection.CollectionID, - Ts: ts, - IsDeleted: true, - } - err = tc.metaDomain.CollChannelDb(txCtx).Insert([]*dbmodel.CollectionChannel{collChannel}) - if err != nil { - log.Error("insert tombstone record for collection_channels failed", zap.String("tenant", tenantID), zap.Int64("collectionID", collection.CollectionID), zap.Uint64("ts", ts), zap.Error(err)) - return err - } - - // 4. insert a mark-deleted record for field_schemas - field := &dbmodel.Field{ - TenantID: tenantID, - CollectionID: collection.CollectionID, - Ts: ts, - IsDeleted: true, - } - err = tc.metaDomain.FieldDb(txCtx).Insert([]*dbmodel.Field{field}) - if err != nil { - log.Error("insert tombstone record for field_schemas failed", zap.String("tenant", tenantID), zap.Int64("collectionID", collection.CollectionID), zap.Uint64("ts", ts), zap.Error(err)) - return err - } - - // 5. insert a mark-deleted record for partitions - partition := &dbmodel.Partition{ - TenantID: tenantID, - CollectionID: collection.CollectionID, - Ts: ts, - IsDeleted: true, - } - err = tc.metaDomain.PartitionDb(txCtx).Insert([]*dbmodel.Partition{partition}) - if err != nil { - log.Error("insert tombstone record for partitions failed", zap.String("tenant", tenantID), zap.Int64("collectionID", collection.CollectionID), zap.Uint64("ts", ts), zap.Error(err)) - return err - } - - return err - }) -} - -func (tc *Catalog) alterModifyCollection(ctx context.Context, oldColl *model.Collection, newColl *model.Collection, ts typeutil.Timestamp) error { - if oldColl.TenantID != newColl.TenantID || oldColl.CollectionID != newColl.CollectionID { - return fmt.Errorf("altering tenant id or collection id is forbidden") - } - - var startPositionsStr string - if newColl.StartPositions != nil { - startPositionsBytes, err := json.Marshal(newColl.StartPositions) - if err != nil { - return fmt.Errorf("failed to marshal start positions: %s", err.Error()) - } - startPositionsStr = string(startPositionsBytes) - } - - properties, err := dbmodel.MarshalProperties(newColl.Properties) - if err != nil { - return err - } - - createdAt, _ := tsoutil.ParseTS(newColl.CreateTime) - tenantID := contextutil.TenantID(ctx) - coll := &dbmodel.Collection{ - TenantID: tenantID, - CollectionID: newColl.CollectionID, - CollectionName: newColl.Name, - Description: newColl.Description, - AutoID: newColl.AutoID, - ShardsNum: newColl.ShardsNum, - StartPosition: startPositionsStr, - ConsistencyLevel: int32(newColl.ConsistencyLevel), - Status: int32(newColl.State), - Ts: ts, - CreatedAt: createdAt, - UpdatedAt: time.Now(), - Properties: properties, - } - - return tc.metaDomain.CollectionDb(ctx).Update(coll) -} - -func (tc *Catalog) AlterCollection(ctx context.Context, oldColl *model.Collection, newColl *model.Collection, alterType metastore.AlterType, ts typeutil.Timestamp) error { - if alterType == metastore.MODIFY { - return tc.alterModifyCollection(ctx, oldColl, newColl, ts) - } - return fmt.Errorf("altering collection doesn't support %s", alterType.String()) -} - -func (tc *Catalog) CreatePartition(ctx context.Context, dbID int64, partition *model.Partition, ts typeutil.Timestamp) error { - tenantID := contextutil.TenantID(ctx) - - p := &dbmodel.Partition{ - TenantID: tenantID, - PartitionID: partition.PartitionID, - PartitionName: partition.PartitionName, - PartitionCreatedTimestamp: partition.PartitionCreatedTimestamp, - CollectionID: partition.CollectionID, - Status: int32(partition.State), - Ts: ts, - } - err := tc.metaDomain.PartitionDb(ctx).Insert([]*dbmodel.Partition{p}) - if err != nil { - log.Error("insert partitions failed", zap.String("tenant", tenantID), zap.Int64("collectionID", partition.CollectionID), zap.Int64("partitionID", partition.PartitionID), zap.Uint64("ts", ts), zap.Error(err)) - return err - } - - return nil -} - -func (tc *Catalog) DropPartition(ctx context.Context, dbID int64, collectionID typeutil.UniqueID, partitionID typeutil.UniqueID, ts typeutil.Timestamp) error { - tenantID := contextutil.TenantID(ctx) - - p := &dbmodel.Partition{ - TenantID: tenantID, - PartitionID: partitionID, - CollectionID: collectionID, - Ts: ts, - IsDeleted: true, - } - err := tc.metaDomain.PartitionDb(ctx).Insert([]*dbmodel.Partition{p}) - if err != nil { - log.Error("insert tombstone record for partition failed", zap.String("tenant", tenantID), zap.Int64("collectionID", collectionID), zap.Int64("partitionID", partitionID), zap.Uint64("ts", ts), zap.Error(err)) - return err - } - - return nil -} - -func (tc *Catalog) alterModifyPartition(ctx context.Context, oldPart *model.Partition, newPart *model.Partition, ts typeutil.Timestamp) error { - createdAt, _ := tsoutil.ParseTS(newPart.PartitionCreatedTimestamp) - p := &dbmodel.Partition{ - TenantID: contextutil.TenantID(ctx), - PartitionID: newPart.PartitionID, - PartitionName: newPart.PartitionName, - PartitionCreatedTimestamp: newPart.PartitionCreatedTimestamp, - CollectionID: newPart.CollectionID, - Status: int32(newPart.State), - Ts: ts, - IsDeleted: false, - CreatedAt: createdAt, - UpdatedAt: time.Now(), - } - return tc.metaDomain.PartitionDb(ctx).Update(p) -} - -func (tc *Catalog) AlterPartition(ctx context.Context, dbID int64, oldPart *model.Partition, newPart *model.Partition, alterType metastore.AlterType, ts typeutil.Timestamp) error { - if alterType == metastore.MODIFY { - return tc.alterModifyPartition(ctx, oldPart, newPart, ts) - } - return fmt.Errorf("altering partition doesn't support: %s", alterType.String()) -} - -func (tc *Catalog) CreateAlias(ctx context.Context, alias *model.Alias, ts typeutil.Timestamp) error { - tenantID := contextutil.TenantID(ctx) - - collAlias := &dbmodel.CollectionAlias{ - TenantID: tenantID, - CollectionID: alias.CollectionID, - CollectionAlias: alias.Name, - Ts: ts, - } - err := tc.metaDomain.CollAliasDb(ctx).Insert([]*dbmodel.CollectionAlias{collAlias}) - if err != nil { - log.Error("insert collection_aliases failed", zap.Int64("collectionID", alias.CollectionID), zap.String("alias", alias.Name), zap.Uint64("ts", ts), zap.Error(err)) - return err - } - - return nil -} - -func (tc *Catalog) DropAlias(ctx context.Context, dbID int64, alias string, ts typeutil.Timestamp) error { - tenantID := contextutil.TenantID(ctx) - - collectionID, err := tc.metaDomain.CollAliasDb(ctx).GetCollectionIDByAlias(tenantID, alias, ts) - if err != nil { - return err - } - - collAlias := &dbmodel.CollectionAlias{ - TenantID: tenantID, - CollectionID: collectionID, - CollectionAlias: alias, - Ts: ts, - IsDeleted: true, - } - err = tc.metaDomain.CollAliasDb(ctx).Insert([]*dbmodel.CollectionAlias{collAlias}) - if err != nil { - log.Error("insert tombstone record for collection_aliases failed", zap.Int64("collectionID", collectionID), zap.String("collAlias", alias), zap.Uint64("ts", ts), zap.Error(err)) - return err - } - - return nil -} - -func (tc *Catalog) AlterAlias(ctx context.Context, alias *model.Alias, ts typeutil.Timestamp) error { - //if ts == 0 { - // tenantID := contextutil.TenantID(ctx) - // alias := collection.Aliases[0] - // - // return tc.metaDomain.CollAliasDb(ctx).Update(tenantID, collection.CollectionID, alias, ts) - //} - - return tc.CreateAlias(ctx, alias, ts) -} - -// ListAliases query collection ID and aliases only, other information are not needed -func (tc *Catalog) ListAliases(ctx context.Context, dbID int64, ts typeutil.Timestamp) ([]*model.Alias, error) { - tenantID := contextutil.TenantID(ctx) - - // 1. find each collection with latest ts - cidTsPairs, err := tc.metaDomain.CollAliasDb(ctx).ListCollectionIDTs(tenantID, ts) - if err != nil { - log.Error("list latest ts and corresponding collectionID in collection_aliases failed", zap.Uint64("ts", ts), zap.Error(err)) - return nil, err - } - if len(cidTsPairs) == 0 { - return []*model.Alias{}, nil - } - - // 2. select with IN clause - collAliases, err := tc.metaDomain.CollAliasDb(ctx).List(tenantID, cidTsPairs) - if err != nil { - log.Error("list collection alias failed", zap.Uint64("ts", ts), zap.Error(err)) - return nil, err - } - - r := make([]*model.Alias, 0, len(collAliases)) - for _, record := range collAliases { - r = append(r, &model.Alias{ - CollectionID: record.CollectionID, - Name: record.CollectionAlias, - }) - } - - return r, nil -} - -func (tc *Catalog) GetCredential(ctx context.Context, username string) (*model.Credential, error) { - tenantID := contextutil.TenantID(ctx) - - user, err := tc.metaDomain.UserDb(ctx).GetByUsername(tenantID, username) - if err != nil { - return nil, err - } - - return dbmodel.UnmarshalUserModel(user), nil -} - -func (tc *Catalog) CreateCredential(ctx context.Context, credential *model.Credential) error { - tenantID := contextutil.TenantID(ctx) - - user := &dbmodel.User{ - TenantID: tenantID, - Username: credential.Username, - EncryptedPassword: credential.EncryptedPassword, - } - - err := tc.metaDomain.UserDb(ctx).Insert(user) - if err != nil { - return err - } - - return nil -} - -func (tc *Catalog) AlterCredential(ctx context.Context, credential *model.Credential) error { - tenantID := contextutil.TenantID(ctx) - - err := tc.metaDomain.UserDb(ctx).UpdatePassword(tenantID, credential.Username, credential.EncryptedPassword) - if err != nil { - return err - } - - return nil -} - -func (tc *Catalog) DropCredential(ctx context.Context, username string) error { - tenantID := contextutil.TenantID(ctx) - - err := tc.metaDomain.UserDb(ctx).MarkDeletedByUsername(tenantID, username) - if err != nil { - return err - } - - return nil -} - -func (tc *Catalog) ListCredentials(ctx context.Context) ([]string, error) { - tenantID := contextutil.TenantID(ctx) - - users, err := tc.metaDomain.UserDb(ctx).ListUser(tenantID) - if err != nil { - return nil, err - } - var usernames []string - for _, user := range users { - usernames = append(usernames, user.Username) - } - return usernames, nil -} - -func (tc *Catalog) CreateRole(ctx context.Context, tenant string, entity *milvuspb.RoleEntity) error { - var err error - if _, err = tc.GetRoleIDByName(ctx, tenant, entity.Name); err != nil && !common.IsKeyNotExistError(err) { - return err - } - if err == nil { - return common.NewIgnorableError(fmt.Errorf("the role[%s] has existed", entity.Name)) - } - return tc.metaDomain.RoleDb(ctx).Insert(&dbmodel.Role{ - Base: dbmodel.Base{TenantID: tenant}, - Name: entity.Name, - }) -} - -func (tc *Catalog) DropRole(ctx context.Context, tenant string, roleName string) error { - return tc.metaDomain.RoleDb(ctx).Delete(tenant, roleName) -} - -func (tc *Catalog) GetRoleIDByName(ctx context.Context, tenant string, name string) (int64, error) { - var ( - roles []*dbmodel.Role - err error - ) - - if roles, err = tc.metaDomain.RoleDb(ctx).GetRoles(tenant, name); err != nil { - return 0, err - } - if len(roles) < 1 { - return 0, common.NewKeyNotExistError(fmt.Sprintf("%s/%s", tenant, name)) - } - return roles[0].ID, nil -} - -func (tc *Catalog) AlterUserRole(ctx context.Context, tenant string, userEntity *milvuspb.UserEntity, roleEntity *milvuspb.RoleEntity, operateType milvuspb.OperateUserRoleType) error { - var ( - user *dbmodel.User - roleID int64 - userRole *dbmodel.UserRole - userRoles []*dbmodel.UserRole - err error - ) - if user, err = tc.metaDomain.UserDb(ctx).GetByUsername(tenant, userEntity.Name); err != nil { - log.Error("fail to get userID by the username", zap.String("username", userEntity.Name), zap.Error(err)) - return err - } - if roleID, err = tc.GetRoleIDByName(ctx, tenant, roleEntity.Name); err != nil { - log.Error("fail to get roleID by the role name", zap.String("role_name", roleEntity.Name), zap.Error(err)) - return err - } - userRole = &dbmodel.UserRole{Base: dbmodel.Base{TenantID: tenant}, UserID: user.ID, RoleID: roleID} - userRoles, err = tc.metaDomain.UserRoleDb(ctx).GetUserRoles(userRole.TenantID, userRole.UserID, userRole.RoleID) - if err != nil { - return err - } - switch operateType { - case milvuspb.OperateUserRoleType_AddUserToRole: - if len(userRoles) > 0 { - return common.NewIgnorableError(fmt.Errorf("the user-role[%s-%s] is existed", userEntity.Name, roleEntity.Name)) - } - return tc.metaDomain.UserRoleDb(ctx).Insert(userRole) - case milvuspb.OperateUserRoleType_RemoveUserFromRole: - if len(userRoles) < 1 { - return common.NewIgnorableError(fmt.Errorf("the user-role[%s-%s] isn't existed", userEntity.Name, roleEntity.Name)) - } - return tc.metaDomain.UserRoleDb(ctx).Delete(userRole.TenantID, userRole.UserID, userRole.RoleID) - default: - err = fmt.Errorf("invalid operate type: %d", operateType) - log.Error("error: ", zap.Error(err)) - return err - } -} - -func (tc *Catalog) ListRole(ctx context.Context, tenant string, entity *milvuspb.RoleEntity, includeUserInfo bool) ([]*milvuspb.RoleResult, error) { - var ( - roleName string - roles []*dbmodel.Role - results []*milvuspb.RoleResult - err error - ) - if entity != nil { - roleName = entity.Name - } - roles, err = tc.metaDomain.RoleDb(ctx).GetRoles(tenant, roleName) - if err != nil { - return nil, err - } - for _, role := range roles { - var users []*milvuspb.UserEntity - var userRoles []*dbmodel.UserRole - if includeUserInfo { - if userRoles, err = tc.metaDomain.UserRoleDb(ctx).GetUserRoles(tenant, 0, role.ID); err != nil { - return nil, err - } - for _, userRole := range userRoles { - users = append(users, &milvuspb.UserEntity{Name: userRole.User.Username}) - } - } - results = append(results, &milvuspb.RoleResult{ - Role: role.Unmarshal(), - Users: users, - }) - } - if !funcutil.IsEmptyString(roleName) && len(results) == 0 { - return nil, common.NewKeyNotExistError(fmt.Sprintf("%s/%s", tenant, roleName)) - } - return results, nil -} - -func (tc *Catalog) ListUser(ctx context.Context, tenant string, entity *milvuspb.UserEntity, includeRoleInfo bool) ([]*milvuspb.UserResult, error) { - var ( - users []*dbmodel.User - results []*milvuspb.UserResult - username string - err error - ) - if entity != nil { - var user *dbmodel.User - username = entity.Name - if user, err = tc.metaDomain.UserDb(ctx).GetByUsername(tenant, username); err != nil { - return nil, err - } - users = append(users, user) - } else { - if users, err = tc.metaDomain.UserDb(ctx).ListUser(tenant); err != nil { - return nil, err - } - } - for _, user := range users { - var roles []*milvuspb.RoleEntity - var userRoles []*dbmodel.UserRole - if includeRoleInfo { - if userRoles, err = tc.metaDomain.UserRoleDb(ctx).GetUserRoles(tenant, user.ID, 0); err != nil { - return nil, err - } - for _, userRole := range userRoles { - roles = append(roles, &milvuspb.RoleEntity{Name: userRole.Role.Name}) - } - } - results = append(results, &milvuspb.UserResult{ - User: &milvuspb.UserEntity{Name: user.Username}, - Roles: roles, - }) - } - return results, nil -} - -func (tc Catalog) grant(ctx context.Context, tenant string, entity *milvuspb.GrantEntity) error { - var ( - roleID int64 - grants []*dbmodel.Grant - grantID int64 - grantIDObjs []*dbmodel.GrantID - err error - ) - if roleID, err = tc.GetRoleIDByName(ctx, tenant, entity.Role.Name); err != nil { - return err - } - - if grants, err = tc.metaDomain.GrantDb(ctx).GetGrants(tenant, roleID, entity.Object.Name, entity.ObjectName); err != nil { - return err - } - if len(grants) == 0 { - var grant = &dbmodel.Grant{ - Base: dbmodel.Base{TenantID: tenant}, - RoleID: roleID, - Object: entity.Object.Name, - ObjectName: entity.ObjectName, - } - if err = tc.metaDomain.GrantDb(ctx).Insert(grant); err != nil { - return err - } - log.Debug("grant id", zap.Int64("id", grant.ID)) - grantID = grant.ID - } else { - grantID = grants[0].ID - } - if grantIDObjs, err = tc.metaDomain.GrantIDDb(ctx).GetGrantIDs(tenant, grantID, entity.Grantor.Privilege.Name, false, false); err != nil { - return err - } - if len(grantIDObjs) > 0 { - log.Warn("the grant id has existed", zap.Any("entity", entity)) - return common.NewIgnorableError(fmt.Errorf("the privilege [%s] has been grantd for the role [%s]", entity.Grantor.Privilege.Name, entity.Role.Name)) - } - var user *dbmodel.User - if user, err = tc.metaDomain.UserDb(ctx).GetByUsername(tenant, entity.Grantor.User.Name); err != nil { - return err - } - return tc.metaDomain.GrantIDDb(ctx).Insert(&dbmodel.GrantID{ - Base: dbmodel.Base{TenantID: tenant}, - GrantID: grantID, - Privilege: entity.Grantor.Privilege.Name, - GrantorID: user.ID, - }) -} - -func (tc Catalog) revoke(ctx context.Context, tenant string, entity *milvuspb.GrantEntity) error { - var ( - roleID int64 - grants []*dbmodel.Grant - grantID int64 - grantIDObjs []*dbmodel.GrantID - err error - ) - - if roleID, err = tc.GetRoleIDByName(ctx, tenant, entity.Role.Name); err != nil { - return err - } - - if grants, err = tc.metaDomain.GrantDb(ctx).GetGrants(tenant, roleID, entity.Object.Name, entity.ObjectName); err != nil { - return err - } - if len(grants) == 0 { - log.Warn("the grant isn't existed", zap.Any("entity", entity)) - return common.NewIgnorableError(fmt.Errorf("the privilege [%s] isn't grantd for the role [%s]", entity.Grantor.Privilege.Name, entity.Role.Name)) - } - grantID = grants[0].ID - if grantIDObjs, err = tc.metaDomain.GrantIDDb(ctx).GetGrantIDs(tenant, grantID, entity.Grantor.Privilege.Name, false, false); err != nil { - return err - } - if len(grantIDObjs) == 0 { - log.Error("the grant-id isn't existed", zap.Any("entity", entity)) - return common.NewIgnorableError(fmt.Errorf("the privilege [%s] isn't grantd for the role [%s]", entity.Grantor.Privilege.Name, entity.Role.Name)) - } - return tc.metaDomain.GrantIDDb(ctx).Delete(tenant, grantID, entity.Grantor.Privilege.Name) -} - -func (tc *Catalog) AlterGrant(ctx context.Context, tenant string, entity *milvuspb.GrantEntity, operateType milvuspb.OperatePrivilegeType) error { - switch operateType { - case milvuspb.OperatePrivilegeType_Grant: - return tc.grant(ctx, tenant, entity) - case milvuspb.OperatePrivilegeType_Revoke: - return tc.revoke(ctx, tenant, entity) - default: - err := fmt.Errorf("invalid operate type: %d", operateType) - log.Error("error: ", zap.Error(err)) - return err - } -} - -func (tc *Catalog) ListGrant(ctx context.Context, tenant string, entity *milvuspb.GrantEntity) ([]*milvuspb.GrantEntity, error) { - var ( - roleID int64 - object string - objectName string - grants []*dbmodel.Grant - grantEntities []*milvuspb.GrantEntity - grantIDDb dbmodel.IGrantIDDb - grantIDs []*dbmodel.GrantID - privilegeName string - err error - ) - if !funcutil.IsEmptyString(entity.ObjectName) && entity.Object != nil && !funcutil.IsEmptyString(entity.Object.Name) { - object = entity.Object.Name - objectName = entity.ObjectName - } - if roleID, err = tc.GetRoleIDByName(ctx, tenant, entity.Role.Name); err != nil { - log.Error("fail to get roleID by the role name", zap.String("role_name", entity.Role.Name), zap.Error(err)) - return nil, err - } - if grants, err = tc.metaDomain.GrantDb(ctx).GetGrants(tenant, roleID, object, objectName); err != nil { - return nil, err - } - grantIDDb = tc.metaDomain.GrantIDDb(ctx) - for _, grant := range grants { - if grantIDs, err = grantIDDb.GetGrantIDs(tenant, grant.ID, "", false, true); err != nil { - return nil, err - } - for _, grantID := range grantIDs { - privilegeName = util.PrivilegeNameForAPI(grantID.Privilege) - if grantID.Privilege == util.AnyWord { - privilegeName = util.AnyWord - } - grantEntities = append(grantEntities, &milvuspb.GrantEntity{ - Role: &milvuspb.RoleEntity{Name: grant.Role.Name}, - Object: &milvuspb.ObjectEntity{Name: grant.Object}, - ObjectName: grant.ObjectName, - Grantor: &milvuspb.GrantorEntity{ - User: &milvuspb.UserEntity{Name: grantID.Grantor.Username}, - Privilege: &milvuspb.PrivilegeEntity{Name: privilegeName}, - }, - }) - } - } - if !funcutil.IsEmptyString(object) && !funcutil.IsEmptyString(objectName) && len(grantEntities) == 0 { - return nil, common.NewKeyNotExistError(fmt.Sprintf("%s/%s/%s/%s", tenant, entity.Role.Name, object, objectName)) - } - return grantEntities, nil -} - -func (tc *Catalog) DeleteGrant(ctx context.Context, tenant string, role *milvuspb.RoleEntity) error { - var ( - roleID int64 - err error - ) - - if roleID, err = tc.GetRoleIDByName(ctx, tenant, role.Name); err != nil { - log.Error("fail to get roleID by the role name", zap.String("role_name", role.Name), zap.Error(err)) - return err - } - return tc.metaDomain.GrantDb(ctx).Delete(tenant, roleID, "", "") -} - -func (tc *Catalog) ListPolicy(ctx context.Context, tenant string) ([]string, error) { - var ( - grants []*dbmodel.Grant - grantIDDb dbmodel.IGrantIDDb - grantIDs []*dbmodel.GrantID - policies []string - err error - ) - if grants, err = tc.metaDomain.GrantDb(ctx).GetGrants(tenant, 0, "", ""); err != nil { - return nil, err - } - grantIDDb = tc.metaDomain.GrantIDDb(ctx) - for _, grant := range grants { - if grantIDs, err = grantIDDb.GetGrantIDs(tenant, grant.ID, "", false, false); err != nil { - return nil, err - } - for _, grantID := range grantIDs { - policies = append(policies, - funcutil.PolicyForPrivilege(grant.Role.Name, grant.Object, grant.ObjectName, grantID.Privilege, "default")) - } - } - - return policies, nil -} - -func (tc *Catalog) ListUserRole(ctx context.Context, tenant string) ([]string, error) { - var ( - userRoleStrs []string - userRoles []*dbmodel.UserRole - err error - ) - - if userRoles, err = tc.metaDomain.UserRoleDb(ctx).GetUserRoles(tenant, 0, 0); err != nil { - return nil, err - } - for _, userRole := range userRoles { - userRoleStrs = append(userRoleStrs, funcutil.EncodeUserRoleCache(userRole.User.Username, userRole.Role.Name)) - } - - return userRoleStrs, nil -} - -func (tc *Catalog) Close() { - -} diff --git a/internal/metastore/db/rootcoord/table_catalog_test.go b/internal/metastore/db/rootcoord/table_catalog_test.go deleted file mode 100644 index 9f1bb2520a..0000000000 --- a/internal/metastore/db/rootcoord/table_catalog_test.go +++ /dev/null @@ -1,2015 +0,0 @@ -package rootcoord - -import ( - "context" - "fmt" - "os" - "testing" - "time" - - "github.com/cockroachdb/errors" - "github.com/stretchr/testify/mock" - "github.com/stretchr/testify/require" - - "github.com/milvus-io/milvus-proto/go-api/v2/commonpb" - "github.com/milvus-io/milvus-proto/go-api/v2/milvuspb" - "github.com/milvus-io/milvus-proto/go-api/v2/schemapb" - "github.com/milvus-io/milvus/internal/metastore" - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel" - "github.com/milvus-io/milvus/internal/metastore/db/dbmodel/mocks" - "github.com/milvus-io/milvus/internal/metastore/model" - pb "github.com/milvus-io/milvus/internal/proto/etcdpb" - "github.com/milvus-io/milvus/pkg/common" - "github.com/milvus-io/milvus/pkg/util" - "github.com/milvus-io/milvus/pkg/util/contextutil" - "github.com/milvus-io/milvus/pkg/util/funcutil" - "github.com/milvus-io/milvus/pkg/util/typeutil" -) - -const ( - tenantID = "test_tenant" - noTs = typeutil.Timestamp(0) - ts = typeutil.Timestamp(10) - collID1 = typeutil.UniqueID(101) - partitionID1 = typeutil.UniqueID(500) - fieldID1 = typeutil.UniqueID(1000) - indexID1 = typeutil.UniqueID(1500) - segmentID1 = typeutil.UniqueID(2000) - indexBuildID1 = typeutil.UniqueID(3000) - - testDb = int64(1000) - - collName1 = "test_collection_name_1" - collAlias1 = "test_collection_alias_1" - collAlias2 = "test_collection_alias_2" - - username = "test_username_1" - password = "test_xxx" -) - -var ( - ctx context.Context - metaDomainMock *mocks.IMetaDomain - collDbMock *mocks.ICollectionDb - fieldDbMock *mocks.IFieldDb - partitionDbMock *mocks.IPartitionDb - collChannelDbMock *mocks.ICollChannelDb - indexDbMock *mocks.IIndexDb - aliasDbMock *mocks.ICollAliasDb - segIndexDbMock *mocks.ISegmentIndexDb - userDbMock *mocks.IUserDb - roleDbMock *mocks.IRoleDb - userRoleDbMock *mocks.IUserRoleDb - grantDbMock *mocks.IGrantDb - grantIDDbMock *mocks.IGrantIDDb - - mockCatalog *Catalog -) - -// TestMain is the first function executed in current package, we will do some initial here -func TestMain(m *testing.M) { - ctx = contextutil.WithTenantID(context.Background(), tenantID) - - collDbMock = &mocks.ICollectionDb{} - fieldDbMock = &mocks.IFieldDb{} - partitionDbMock = &mocks.IPartitionDb{} - collChannelDbMock = &mocks.ICollChannelDb{} - indexDbMock = &mocks.IIndexDb{} - aliasDbMock = &mocks.ICollAliasDb{} - segIndexDbMock = &mocks.ISegmentIndexDb{} - userDbMock = &mocks.IUserDb{} - roleDbMock = &mocks.IRoleDb{} - userRoleDbMock = &mocks.IUserRoleDb{} - grantDbMock = &mocks.IGrantDb{} - grantIDDbMock = &mocks.IGrantIDDb{} - - metaDomainMock = &mocks.IMetaDomain{} - metaDomainMock.On("CollectionDb", ctx).Return(collDbMock) - metaDomainMock.On("FieldDb", ctx).Return(fieldDbMock) - metaDomainMock.On("PartitionDb", ctx).Return(partitionDbMock) - metaDomainMock.On("CollChannelDb", ctx).Return(collChannelDbMock) - metaDomainMock.On("IndexDb", ctx).Return(indexDbMock) - metaDomainMock.On("CollAliasDb", ctx).Return(aliasDbMock) - metaDomainMock.On("SegmentIndexDb", ctx).Return(segIndexDbMock) - metaDomainMock.On("UserDb", ctx).Return(userDbMock) - metaDomainMock.On("RoleDb", ctx).Return(roleDbMock) - metaDomainMock.On("UserRoleDb", ctx).Return(userRoleDbMock) - metaDomainMock.On("GrantDb", ctx).Return(grantDbMock) - metaDomainMock.On("GrantIDDb", ctx).Return(grantIDDbMock) - - mockCatalog = mockMetaCatalog(metaDomainMock) - - // m.Run entry for executing tests - os.Exit(m.Run()) -} - -type NoopTransaction struct{} - -func (*NoopTransaction) Transaction(ctx context.Context, fn func(txctx context.Context) error) error { - return fn(ctx) -} - -func mockMetaCatalog(petDomain dbmodel.IMetaDomain) *Catalog { - return NewTableCatalog(&NoopTransaction{}, petDomain) -} - -func TestTableCatalog_CreateCollection(t *testing.T) { - coll := &model.Collection{ - CollectionID: collID1, - Name: collName1, - AutoID: false, - Fields: []*model.Field{ - { - FieldID: fieldID1, - Name: "test_field_name_1", - IsPrimaryKey: false, - Description: "", - DataType: schemapb.DataType_FloatVector, - TypeParams: []*commonpb.KeyValuePair{ - { - Key: "test_type_params_k1", - Value: "test_type_params_v1", - }, - { - Key: "test_type_params_k2", - Value: "test_type_params_v2", - }, - }, - IndexParams: []*commonpb.KeyValuePair{ - { - Key: "test_index_params_k1", - Value: "test_index_params_v1", - }, - { - Key: "test_index_params_k2", - Value: "test_index_params_v2", - }, - }, - }, - }, - StartPositions: []*commonpb.KeyDataPair{ - { - Key: "test_start_position_key1", - Data: []byte("test_start_position_data1"), - }, - }, - CreateTime: 0, - Partitions: []*model.Partition{ - { - PartitionID: partitionID1, - PartitionName: "test_partition_name_1", - PartitionCreatedTimestamp: 0, - }, - }, - VirtualChannelNames: []string{ - fmt.Sprintf("dmChannel_%dv%d", collID1, 0), - fmt.Sprintf("dmChannel_%dv%d", collID1, 1), - }, - PhysicalChannelNames: []string{ - funcutil.ToPhysicalChannel(fmt.Sprintf("dmChannel_%dv%d", collID1, 0)), - funcutil.ToPhysicalChannel(fmt.Sprintf("dmChannel_%dv%d", collID1, 1)), - }, - } - - // expectation - collDbMock.On("Insert", mock.Anything).Return(nil).Once() - fieldDbMock.On("Insert", mock.Anything).Return(nil).Once() - partitionDbMock.On("Insert", mock.Anything).Return(nil).Once() - collChannelDbMock.On("Insert", mock.Anything).Return(nil).Once() - - // actual - gotErr := mockCatalog.CreateCollection(ctx, coll, ts) - require.Equal(t, nil, gotErr) -} - -func TestTableCatalog_CreateCollection_InsertCollError(t *testing.T) { - coll := &model.Collection{ - CollectionID: collID1, - Name: collName1, - AutoID: false, - } - - // expectation - errTest := errors.New("test error") - collDbMock.On("Insert", mock.Anything).Return(errTest).Once() - - // actual - gotErr := mockCatalog.CreateCollection(ctx, coll, ts) - require.Equal(t, errTest, gotErr) -} - -func TestTableCatalog_CreateCollection_MarshalStartPositionsError(t *testing.T) { - coll := &model.Collection{ - CollectionID: collID1, - Name: collName1, - AutoID: false, - StartPositions: []*commonpb.KeyDataPair{ - { - Key: "test_start_position_key1", - Data: []byte("\\u002"), - }, - }, - } - - // expectation - errTest := errors.New("test error") - collDbMock.On("Insert", mock.Anything).Return(errTest).Once() - - // actual - gotErr := mockCatalog.CreateCollection(ctx, coll, ts) - require.Equal(t, errTest, gotErr) -} - -func TestTableCatalog_GetCollectionByID(t *testing.T) { - coll := &dbmodel.Collection{ - TenantID: tenantID, - CollectionID: collID1, - CollectionName: collName1, - AutoID: true, - StartPosition: "", - Ts: ts, - } - fields := []*dbmodel.Field{ - { - FieldID: fieldID1, - FieldName: "test_field_name_1", - IsPrimaryKey: false, - Description: "", - DataType: schemapb.DataType_FloatVector, - TypeParams: "", - IndexParams: "", - }, - } - partitions := []*dbmodel.Partition{ - { - PartitionID: partitionID1, - PartitionName: "test_partition_name_1", - PartitionCreatedTimestamp: 0, - }, - } - collChannels := []*dbmodel.CollectionChannel{ - { - TenantID: tenantID, - CollectionID: collID1, - VirtualChannelName: "test_virtual_channel_name_1", - PhysicalChannelName: "test_physical_channel_name_1", - }, - } - indexes := []*dbmodel.Index{ - { - TenantID: tenantID, - FieldID: fieldID1, - CollectionID: collID1, - IndexID: indexID1, - IndexName: "test_index_name_1", - IndexParams: "", - }, - } - - // expectation - collDbMock.On("GetCollectionIDTs", tenantID, collID1, ts).Return(&dbmodel.Collection{CollectionID: collID1, Ts: ts}, nil).Once() - collDbMock.On("Get", tenantID, collID1, ts).Return(coll, nil).Once() - fieldDbMock.On("GetByCollectionID", tenantID, collID1, ts).Return(fields, nil).Once() - partitionDbMock.On("GetByCollectionID", tenantID, collID1, ts).Return(partitions, nil).Once() - collChannelDbMock.On("GetByCollectionID", tenantID, collID1, ts).Return(collChannels, nil).Once() - indexDbMock.On("Get", tenantID, collID1).Return(indexes, nil).Once() - - // actual - res, gotErr := mockCatalog.GetCollectionByID(ctx, util.NonDBID, ts, collID1) - // collection basic info - require.Equal(t, nil, gotErr) - require.Equal(t, coll.TenantID, res.TenantID) - require.Equal(t, coll.CollectionID, res.CollectionID) - require.Equal(t, coll.CollectionName, res.Name) - require.Equal(t, coll.AutoID, res.AutoID) - require.Equal(t, coll.Ts, res.CreateTime) - require.Empty(t, res.StartPositions) - // partitions/fields/channels - require.NotEmpty(t, res.Partitions) - require.NotEmpty(t, res.Fields) - require.NotEmpty(t, res.VirtualChannelNames) - require.NotEmpty(t, res.PhysicalChannelNames) -} - -func TestTableCatalog_GetCollectionByID_UnmarshalStartPositionsError(t *testing.T) { - coll := &dbmodel.Collection{ - TenantID: tenantID, - CollectionID: collID1, - CollectionName: collName1, - AutoID: false, - StartPosition: "\"Key\": \"test_start_position_key1\",\"Data\": \"test_start_position_data1\",", - CreatedAt: time.UnixMilli(10000), - } - - // expectation - collDbMock.On("GetCollectionIDTs", tenantID, collID1, ts).Return(&dbmodel.Collection{CollectionID: collID1, Ts: ts}, nil).Once() - collDbMock.On("Get", tenantID, collID1, ts).Return(coll, nil).Once() - fieldDbMock.On("GetByCollectionID", tenantID, collID1, ts).Return(nil, nil).Once() - partitionDbMock.On("GetByCollectionID", tenantID, collID1, ts).Return(nil, nil).Once() - collChannelDbMock.On("GetByCollectionID", tenantID, collID1, ts).Return(nil, nil).Once() - indexDbMock.On("Get", tenantID, collID1).Return(nil, nil).Once() - - // actual - res, gotErr := mockCatalog.GetCollectionByID(ctx, util.NonDBID, ts, collID1) - require.Nil(t, res) - require.Error(t, gotErr) -} - -func TestTableCatalog_GetCollectionByID_SelectCollError(t *testing.T) { - // expectation - errTest := errors.New("select collection error") - collDbMock.On("GetCollectionIDTs", tenantID, collID1, ts).Return(&dbmodel.Collection{CollectionID: collID1, Ts: ts}, nil).Once() - collDbMock.On("Get", tenantID, collID1, ts).Return(nil, errTest).Once() - - // actual - res, gotErr := mockCatalog.GetCollectionByID(ctx, util.NonDBID, ts, collID1) - require.Nil(t, res) - require.Error(t, gotErr) -} - -func TestTableCatalog_GetCollectionByID_SelectFieldError(t *testing.T) { - coll := &dbmodel.Collection{ - TenantID: tenantID, - CollectionID: collID1, - CollectionName: collName1, - AutoID: false, - StartPosition: "", - CreatedAt: time.UnixMilli(10000), - } - - // expectation - errTest := errors.New("select fields error") - collDbMock.On("GetCollectionIDTs", tenantID, collID1, ts).Return(&dbmodel.Collection{CollectionID: collID1, Ts: ts}, nil).Once() - collDbMock.On("Get", tenantID, collID1, ts).Return(coll, nil).Once() - fieldDbMock.On("GetByCollectionID", tenantID, collID1, ts).Return(nil, errTest).Once() - - // actual - res, gotErr := mockCatalog.GetCollectionByID(ctx, util.NonDBID, ts, collID1) - require.Nil(t, res) - require.Error(t, gotErr) -} - -func TestTableCatalog_GetCollectionByID_SelectPartitionError(t *testing.T) { - coll := &dbmodel.Collection{ - TenantID: tenantID, - CollectionID: collID1, - CollectionName: collName1, - AutoID: false, - StartPosition: "", - CreatedAt: time.UnixMilli(10000), - } - - // expectation - errTest := errors.New("select fields error") - collDbMock.On("GetCollectionIDTs", tenantID, collID1, ts).Return(&dbmodel.Collection{CollectionID: collID1, Ts: ts}, nil).Once() - collDbMock.On("Get", tenantID, collID1, ts).Return(coll, nil).Once() - fieldDbMock.On("GetByCollectionID", tenantID, collID1, ts).Return(nil, nil).Once() - partitionDbMock.On("GetByCollectionID", tenantID, collID1, ts).Return(nil, errTest).Once() - - // actual - res, gotErr := mockCatalog.GetCollectionByID(ctx, util.NonDBID, ts, collID1) - require.Nil(t, res) - require.Error(t, gotErr) -} - -func TestTableCatalog_GetCollectionByID_SelectChannelError(t *testing.T) { - coll := &dbmodel.Collection{ - TenantID: tenantID, - CollectionID: collID1, - CollectionName: collName1, - AutoID: false, - StartPosition: "", - CreatedAt: time.UnixMilli(10000), - } - - // expectation - errTest := errors.New("select fields error") - collDbMock.On("GetCollectionIDTs", tenantID, collID1, ts).Return(&dbmodel.Collection{CollectionID: collID1, Ts: ts}, nil).Once() - collDbMock.On("Get", tenantID, collID1, ts).Return(coll, nil).Once() - fieldDbMock.On("GetByCollectionID", tenantID, collID1, ts).Return(nil, nil).Once() - partitionDbMock.On("GetByCollectionID", tenantID, collID1, ts).Return(nil, nil).Once() - collChannelDbMock.On("GetByCollectionID", tenantID, collID1, ts).Return(nil, errTest).Once() - - // actual - res, gotErr := mockCatalog.GetCollectionByID(ctx, util.NonDBID, ts, collID1) - require.Nil(t, res) - require.Error(t, gotErr) -} - -func TestTableCatalog_GetCollectionByName(t *testing.T) { - coll := &dbmodel.Collection{ - TenantID: tenantID, - CollectionID: collID1, - CollectionName: collName1, - AutoID: true, - StartPosition: "", - Ts: ts, - } - fields := []*dbmodel.Field{ - { - FieldID: fieldID1, - FieldName: "test_field_name_1", - IsPrimaryKey: false, - Description: "", - DataType: schemapb.DataType_FloatVector, - TypeParams: "", - IndexParams: "", - }, - } - partitions := []*dbmodel.Partition{ - { - PartitionID: partitionID1, - PartitionName: "test_partition_name_1", - PartitionCreatedTimestamp: 0, - }, - } - collChannels := []*dbmodel.CollectionChannel{ - { - TenantID: tenantID, - CollectionID: collID1, - VirtualChannelName: "test_virtual_channel_name_1", - PhysicalChannelName: "test_physical_channel_name_1", - }, - } - indexes := []*dbmodel.Index{ - { - TenantID: tenantID, - FieldID: fieldID1, - CollectionID: collID1, - IndexID: indexID1, - IndexName: "test_index_name_1", - IndexParams: "", - }, - } - - // expectation - collDbMock.On("GetCollectionIDByName", tenantID, collName1, ts).Return(collID1, nil).Once() - collDbMock.On("GetCollectionIDTs", tenantID, collID1, ts).Return(&dbmodel.Collection{CollectionID: collID1, Ts: ts}, nil).Once() - collDbMock.On("Get", tenantID, collID1, ts).Return(coll, nil).Once() - fieldDbMock.On("GetByCollectionID", tenantID, collID1, ts).Return(fields, nil).Once() - partitionDbMock.On("GetByCollectionID", tenantID, collID1, ts).Return(partitions, nil).Once() - collChannelDbMock.On("GetByCollectionID", tenantID, collID1, ts).Return(collChannels, nil).Once() - indexDbMock.On("Get", tenantID, collID1).Return(indexes, nil).Once() - - // actual - res, gotErr := mockCatalog.GetCollectionByName(ctx, util.NonDBID, collName1, ts) - // collection basic info - require.Equal(t, nil, gotErr) - require.Equal(t, coll.TenantID, res.TenantID) - require.Equal(t, coll.CollectionID, res.CollectionID) - require.Equal(t, coll.CollectionName, res.Name) - require.Equal(t, coll.AutoID, res.AutoID) - require.Equal(t, coll.Ts, res.CreateTime) - require.Empty(t, res.StartPositions) - // partitions/fields/channels - require.NotEmpty(t, res.Partitions) - require.NotEmpty(t, res.Fields) - require.NotEmpty(t, res.VirtualChannelNames) - require.NotEmpty(t, res.PhysicalChannelNames) -} - -func TestTableCatalog_GetCollectionByName_SelectCollIDError(t *testing.T) { - // expectation - errTest := errors.New("select fields error") - collDbMock.On("GetCollectionIDByName", tenantID, collName1, ts).Return(typeutil.UniqueID(0), errTest).Once() - - // actual - res, gotErr := mockCatalog.GetCollectionByName(ctx, util.NonDBID, collName1, ts) - require.Nil(t, res) - require.Error(t, gotErr) -} - -func TestTableCatalog_ListCollections(t *testing.T) { - coll := &dbmodel.Collection{ - TenantID: tenantID, - CollectionID: collID1, - CollectionName: collName1, - AutoID: true, - StartPosition: "", - Ts: ts, - } - fields := []*dbmodel.Field{ - { - FieldID: fieldID1, - FieldName: "test_field_name_1", - IsPrimaryKey: false, - Description: "", - DataType: schemapb.DataType_FloatVector, - TypeParams: "", - IndexParams: "", - }, - } - partitions := []*dbmodel.Partition{ - { - PartitionID: partitionID1, - PartitionName: "test_partition_name_1", - PartitionCreatedTimestamp: 0, - }, - } - collChannels := []*dbmodel.CollectionChannel{ - { - TenantID: tenantID, - CollectionID: collID1, - VirtualChannelName: "test_virtual_channel_name_1", - PhysicalChannelName: "test_physical_channel_name_1", - }, - } - indexes := []*dbmodel.Index{ - { - TenantID: tenantID, - FieldID: fieldID1, - CollectionID: collID1, - IndexID: indexID1, - IndexName: "test_index_name_1", - IndexParams: "", - }, - } - - // expectation - collDbMock.On("ListCollectionIDTs", tenantID, ts).Return([]*dbmodel.Collection{{CollectionID: collID1, Ts: ts}}, nil).Once() - collDbMock.On("Get", tenantID, collID1, ts).Return(coll, nil).Once() - fieldDbMock.On("GetByCollectionID", tenantID, collID1, ts).Return(fields, nil).Once() - partitionDbMock.On("GetByCollectionID", tenantID, collID1, ts).Return(partitions, nil).Once() - collChannelDbMock.On("GetByCollectionID", tenantID, collID1, ts).Return(collChannels, nil).Once() - indexDbMock.On("Get", tenantID, collID1).Return(indexes, nil).Once() - - // actual - res, gotErr := mockCatalog.ListCollections(ctx, util.NonDBID, ts) - // collection basic info - require.Equal(t, nil, gotErr) - require.Equal(t, 1, len(res)) - require.Equal(t, coll.TenantID, res[0].TenantID) - require.Equal(t, coll.CollectionID, res[0].CollectionID) - require.Equal(t, coll.CollectionName, res[0].Name) - require.Equal(t, coll.AutoID, res[0].AutoID) - require.Equal(t, coll.Ts, res[0].CreateTime) - require.Empty(t, res[0].StartPositions) - // partitions/fields/channels - require.NotEmpty(t, res[0].Partitions) - require.NotEmpty(t, res[0].Fields) - require.NotEmpty(t, res[0].VirtualChannelNames) - require.NotEmpty(t, res[0].PhysicalChannelNames) -} - -func TestTableCatalog_CollectionExists(t *testing.T) { - resultTs := typeutil.Timestamp(5) - coll := &dbmodel.Collection{ - TenantID: tenantID, - CollectionID: collID1, - CollectionName: collName1, - } - - // expectation - collDbMock.On("GetCollectionIDTs", tenantID, collID1, ts).Return(&dbmodel.Collection{CollectionID: collID1, Ts: resultTs}, nil).Once() - collDbMock.On("Get", tenantID, collID1, resultTs).Return(coll, nil).Once() - - // actual - res := mockCatalog.CollectionExists(ctx, util.NonDBID, collID1, ts) - require.True(t, res) -} - -func TestTableCatalog_CollectionExists_IsDeletedTrue(t *testing.T) { - resultTs := typeutil.Timestamp(5) - coll := &dbmodel.Collection{ - TenantID: tenantID, - CollectionID: collID1, - CollectionName: collName1, - IsDeleted: true, - } - - // expectation - collDbMock.On("GetCollectionIDTs", tenantID, collID1, ts).Return(&dbmodel.Collection{CollectionID: collID1, Ts: resultTs}, nil).Once() - collDbMock.On("Get", tenantID, collID1, resultTs).Return(coll, nil).Once() - - // actual - res := mockCatalog.CollectionExists(ctx, util.NonDBID, collID1, ts) - require.False(t, res) -} - -func TestTableCatalog_CollectionExists_CollNotExists(t *testing.T) { - resultTs := typeutil.Timestamp(5) - - // expectation - collDbMock.On("GetCollectionIDTs", tenantID, collID1, ts).Return(&dbmodel.Collection{CollectionID: collID1, Ts: resultTs}, nil).Once() - collDbMock.On("Get", tenantID, collID1, resultTs).Return(nil, nil).Once() - - // actual - res := mockCatalog.CollectionExists(ctx, util.NonDBID, collID1, ts) - require.False(t, res) -} - -func TestTableCatalog_CollectionExists_GetCidTsError(t *testing.T) { - // expectation - errTest := errors.New("select error") - collDbMock.On("GetCollectionIDTs", tenantID, collID1, ts).Return(nil, errTest).Once() - - // actual - res := mockCatalog.CollectionExists(ctx, util.NonDBID, collID1, ts) - require.False(t, res) -} - -func TestTableCatalog_DropCollection_TsNot0(t *testing.T) { - coll := &model.Collection{ - TenantID: tenantID, - CollectionID: collID1, - Name: collName1, - Aliases: []string{collAlias1, collAlias2}, - } - inColl := &dbmodel.Collection{ - TenantID: tenantID, - CollectionID: coll.CollectionID, - Ts: ts, - IsDeleted: true, - } - inAliases := []*dbmodel.CollectionAlias{ - { - TenantID: tenantID, - CollectionID: coll.CollectionID, - CollectionAlias: coll.Aliases[0], - Ts: ts, - IsDeleted: true, - }, - { - TenantID: tenantID, - CollectionID: coll.CollectionID, - CollectionAlias: coll.Aliases[1], - Ts: ts, - IsDeleted: true, - }, - } - inChannels := []*dbmodel.CollectionChannel{ - { - TenantID: tenantID, - CollectionID: coll.CollectionID, - Ts: ts, - IsDeleted: true, - }, - } - inFields := []*dbmodel.Field{ - { - TenantID: tenantID, - CollectionID: coll.CollectionID, - Ts: ts, - IsDeleted: true, - }, - } - inPartitions := []*dbmodel.Partition{ - { - TenantID: tenantID, - CollectionID: coll.CollectionID, - Ts: ts, - IsDeleted: true, - }, - } - - // expectation - collDbMock.On("Insert", inColl).Return(nil).Once() - aliasDbMock.On("Insert", inAliases).Return(nil).Once() - collChannelDbMock.On("Insert", inChannels).Return(nil).Once() - fieldDbMock.On("Insert", inFields).Return(nil).Once() - partitionDbMock.On("Insert", inPartitions).Return(nil).Once() - indexDbMock.On("MarkDeletedByCollectionID", tenantID, coll.CollectionID).Return(nil).Once() - segIndexDbMock.On("MarkDeletedByCollectionID", tenantID, coll.CollectionID).Return(nil).Once() - - // actual - gotErr := mockCatalog.DropCollection(ctx, coll, ts) - require.NoError(t, gotErr) -} - -func TestTableCatalog_DropCollection_TsNot0_CollInsertError(t *testing.T) { - coll := &model.Collection{ - TenantID: tenantID, - CollectionID: collID1, - Name: collName1, - Aliases: []string{collAlias1, collAlias2}, - } - - // expectation - errTest := errors.New("test error") - collDbMock.On("Insert", mock.Anything).Return(errTest).Once() - - // actual - gotErr := mockCatalog.DropCollection(ctx, coll, ts) - require.Error(t, gotErr) -} - -func TestTableCatalog_DropCollection_TsNot0_AliasInsertError(t *testing.T) { - coll := &model.Collection{ - TenantID: tenantID, - CollectionID: collID1, - Name: collName1, - Aliases: []string{collAlias1, collAlias2}, - } - - // expectation - errTest := errors.New("test error") - collDbMock.On("Insert", mock.Anything).Return(nil).Once() - aliasDbMock.On("Insert", mock.Anything).Return(errTest).Once() - - // actual - gotErr := mockCatalog.DropCollection(ctx, coll, ts) - require.Error(t, gotErr) -} - -func TestTableCatalog_DropCollection_TsNot0_ChannelInsertError(t *testing.T) { - coll := &model.Collection{ - TenantID: tenantID, - CollectionID: collID1, - Name: collName1, - Aliases: []string{collAlias1, collAlias2}, - } - - // expectation - errTest := errors.New("test error") - collDbMock.On("Insert", mock.Anything).Return(nil).Once() - aliasDbMock.On("Insert", mock.Anything).Return(nil).Once() - collChannelDbMock.On("Insert", mock.Anything).Return(errTest).Once() - - // actual - gotErr := mockCatalog.DropCollection(ctx, coll, ts) - require.Error(t, gotErr) -} - -func TestTableCatalog_DropCollection_TsNot0_FieldInsertError(t *testing.T) { - coll := &model.Collection{ - TenantID: tenantID, - CollectionID: collID1, - Name: collName1, - Aliases: []string{collAlias1, collAlias2}, - } - - // expectation - errTest := errors.New("test error") - collDbMock.On("Insert", mock.Anything).Return(nil).Once() - aliasDbMock.On("Insert", mock.Anything).Return(nil).Once() - collChannelDbMock.On("Insert", mock.Anything).Return(nil).Once() - fieldDbMock.On("Insert", mock.Anything).Return(errTest).Once() - - // actual - gotErr := mockCatalog.DropCollection(ctx, coll, ts) - require.Error(t, gotErr) -} - -func TestTableCatalog_DropCollection_TsNot0_PartitionInsertError(t *testing.T) { - coll := &model.Collection{ - TenantID: tenantID, - CollectionID: collID1, - Name: collName1, - Aliases: []string{collAlias1, collAlias2}, - } - - // expectation - errTest := errors.New("test error") - collDbMock.On("Insert", mock.Anything).Return(nil).Once() - aliasDbMock.On("Insert", mock.Anything).Return(nil).Once() - collChannelDbMock.On("Insert", mock.Anything).Return(nil).Once() - fieldDbMock.On("Insert", mock.Anything).Return(nil).Once() - partitionDbMock.On("Insert", mock.Anything).Return(errTest).Once() - - // actual - gotErr := mockCatalog.DropCollection(ctx, coll, ts) - require.Error(t, gotErr) -} - -func TestCatalog_AlterCollection(t *testing.T) { - coll := &model.Collection{ - TenantID: tenantID, - CollectionID: collID1, - Name: collName1, - State: pb.CollectionState_CollectionCreated, - Aliases: []string{collAlias1, collAlias2}, - } - newColl := &model.Collection{ - TenantID: tenantID, - CollectionID: collID1, - Name: collName1, - State: pb.CollectionState_CollectionDropping, - Aliases: []string{collAlias1, collAlias2}, - } - - collDbMock.On("Update", mock.Anything).Return(nil).Once() - - gotErr := mockCatalog.AlterCollection(ctx, coll, newColl, metastore.MODIFY, ts) - require.NoError(t, gotErr) -} - -func TestTableCatalog_AlterCollection_TsNot0_AlterTypeError(t *testing.T) { - coll := &model.Collection{ - TenantID: tenantID, - CollectionID: collID1, - Name: collName1, - State: pb.CollectionState_CollectionCreated, - Aliases: []string{collAlias1, collAlias2}, - } - - gotErr := mockCatalog.AlterCollection(ctx, coll, coll, metastore.ADD, ts) - require.Error(t, gotErr) - - gotErr = mockCatalog.AlterCollection(ctx, coll, coll, metastore.DELETE, ts) - require.Error(t, gotErr) -} - -func TestCatalog_AlterCollection_TsNot0_CollInsertError(t *testing.T) { - coll := &model.Collection{ - TenantID: tenantID, - CollectionID: collID1, - Name: collName1, - State: pb.CollectionState_CollectionCreated, - Aliases: []string{collAlias1, collAlias2}, - } - - // expectation - errTest := errors.New("test error") - collDbMock.On("Update", mock.Anything).Return(errTest).Once() - - // actual - gotErr := mockCatalog.AlterCollection(ctx, coll, coll, metastore.MODIFY, ts) - require.Error(t, gotErr) -} - -func TestTableCatalog_CreatePartition(t *testing.T) { - partition := &model.Partition{ - PartitionID: partitionID1, - PartitionName: "test_partition_name_1", - PartitionCreatedTimestamp: 0, - CollectionID: collID1, - } - - // expectation - partitionDbMock.On("Insert", mock.Anything).Return(nil).Once() - - // actual - gotErr := mockCatalog.CreatePartition(ctx, util.NonDBID, partition, ts) - require.Equal(t, nil, gotErr) -} - -func TestTableCatalog_CreatePartition_InsertPartitionError(t *testing.T) { - partition := &model.Partition{ - PartitionID: partitionID1, - PartitionName: "test_partition_name_1", - PartitionCreatedTimestamp: 0, - CollectionID: collID1, - } - - // expectation - errTest := errors.New("test error") - partitionDbMock.On("Insert", mock.Anything).Return(errTest).Once() - - // actual - gotErr := mockCatalog.CreatePartition(ctx, util.NonDBID, partition, ts) - require.Error(t, gotErr) -} - -func TestTableCatalog_DropPartition_TsNot0(t *testing.T) { - // expectation - partitionDbMock.On("Insert", mock.Anything).Return(nil).Once() - - // actual - gotErr := mockCatalog.DropPartition(ctx, util.NonDBID, collID1, partitionID1, ts) - require.NoError(t, gotErr) -} - -func TestTableCatalog_DropPartition_TsNot0_PartitionInsertError(t *testing.T) { - // expectation - errTest := errors.New("test error") - partitionDbMock.On("Insert", mock.Anything).Return(errTest).Once() - - // actual - gotErr := mockCatalog.DropPartition(ctx, util.NonDBID, collID1, partitionID1, ts) - require.Error(t, gotErr) -} - -func TestCatalog_AlterPartition(t *testing.T) { - partition := &model.Partition{ - PartitionID: partitionID1, - PartitionName: "test_partition_name_1", - PartitionCreatedTimestamp: 1, - CollectionID: collID1, - State: pb.PartitionState_PartitionCreated, - } - newPartition := &model.Partition{ - PartitionID: partitionID1, - PartitionName: "test_partition_name_1", - PartitionCreatedTimestamp: 1, - CollectionID: collID1, - State: pb.PartitionState_PartitionDropping, - } - - partitionDbMock.On("Update", mock.Anything).Return(nil).Once() - - gotErr := mockCatalog.AlterPartition(ctx, util.NonDBID, partition, newPartition, metastore.MODIFY, ts) - require.NoError(t, gotErr) -} - -func TestCatalog_AlterPartition_TsNot0_AlterTypeError(t *testing.T) { - partition := &model.Partition{ - PartitionID: partitionID1, - PartitionName: "test_partition_name_1", - PartitionCreatedTimestamp: 1, - CollectionID: collID1, - State: pb.PartitionState_PartitionCreated, - } - - gotErr := mockCatalog.AlterPartition(ctx, util.NonDBID, partition, partition, metastore.ADD, ts) - require.Error(t, gotErr) - - gotErr = mockCatalog.AlterPartition(ctx, util.NonDBID, partition, partition, metastore.DELETE, ts) - require.Error(t, gotErr) -} - -func TestCatalog_AlterPartition_TsNot0_PartitionInsertError(t *testing.T) { - partition := &model.Partition{ - PartitionID: partitionID1, - PartitionName: "test_partition_name_1", - PartitionCreatedTimestamp: 1, - CollectionID: collID1, - State: pb.PartitionState_PartitionCreated, - } - - // expectation - errTest := errors.New("test error") - partitionDbMock.On("Update", mock.Anything).Return(errTest).Once() - - // actual - gotErr := mockCatalog.AlterPartition(ctx, util.NonDBID, partition, partition, metastore.MODIFY, ts) - require.Error(t, gotErr) -} - -func TestTableCatalog_CreateAlias(t *testing.T) { - alias := &model.Alias{ - CollectionID: collID1, - Name: collAlias1, - } - - // expectation - aliasDbMock.On("Insert", mock.Anything).Return(nil).Once() - - // actual - gotErr := mockCatalog.CreateAlias(ctx, alias, ts) - require.NoError(t, gotErr) -} - -func TestTableCatalog_CreateAlias_InsertAliasError(t *testing.T) { - alias := &model.Alias{ - CollectionID: collID1, - Name: collAlias1, - } - - // expectation - errTest := errors.New("test error") - aliasDbMock.On("Insert", mock.Anything).Return(errTest).Once() - - // actual - gotErr := mockCatalog.CreateAlias(ctx, alias, ts) - require.Error(t, gotErr) -} - -func TestTableCatalog_DropAlias_TsNot0(t *testing.T) { - // expectation - aliasDbMock.On("GetCollectionIDByAlias", tenantID, collAlias1, ts).Return(collID1, nil).Once() - aliasDbMock.On("Insert", mock.Anything).Return(nil).Once() - - // actual - gotErr := mockCatalog.DropAlias(ctx, testDb, collAlias1, ts) - require.NoError(t, gotErr) -} - -func TestTableCatalog_DropAlias_TsNot0_SelectCollectionIDByAliasError(t *testing.T) { - // expectation - errTest := errors.New("test error") - aliasDbMock.On("GetCollectionIDByAlias", tenantID, collAlias1, ts).Return(typeutil.UniqueID(0), errTest).Once() - - // actual - gotErr := mockCatalog.DropAlias(ctx, testDb, collAlias1, ts) - require.Error(t, gotErr) -} - -func TestTableCatalog_DropAlias_TsNot0_InsertIndexError(t *testing.T) { - // expectation - errTest := errors.New("test error") - aliasDbMock.On("GetCollectionIDByAlias", tenantID, collAlias1, ts).Return(collID1, nil).Once() - aliasDbMock.On("Insert", mock.Anything).Return(errTest).Once() - - // actual - gotErr := mockCatalog.DropAlias(ctx, testDb, collAlias1, ts) - require.Error(t, gotErr) -} - -func TestTableCatalog_AlterAlias_TsNot0(t *testing.T) { - alias := &model.Alias{ - CollectionID: collID1, - Name: collAlias1, - } - - // expectation - aliasDbMock.On("Insert", mock.Anything).Return(nil).Once() - - // actual - gotErr := mockCatalog.AlterAlias(ctx, alias, ts) - require.NoError(t, gotErr) -} - -func TestTableCatalog_ListAliases(t *testing.T) { - out := []*model.Alias{ - { - CollectionID: collID1, - Name: collAlias1, - }, - } - collAliases := []*dbmodel.CollectionAlias{ - { - CollectionID: collID1, - CollectionAlias: collAlias1, - }, - } - - // expectation - cidTsPairs := []*dbmodel.CollectionAlias{{CollectionID: collID1, Ts: ts}} - aliasDbMock.On("ListCollectionIDTs", tenantID, ts).Return(cidTsPairs, nil).Once() - aliasDbMock.On("List", tenantID, cidTsPairs).Return(collAliases, nil).Once() - - // actual - res, gotErr := mockCatalog.ListAliases(ctx, testDb, ts) - require.Equal(t, nil, gotErr) - require.Equal(t, out, res) -} - -func TestTableCatalog_ListAliases_NoResult(t *testing.T) { - // expectation - aliasDbMock.On("ListCollectionIDTs", tenantID, ts).Return(nil, nil).Once() - - // actual - res, gotErr := mockCatalog.ListAliases(ctx, testDb, ts) - require.Equal(t, nil, gotErr) - require.Empty(t, res) -} - -func TestTableCatalog_ListAliases_ListCidTsError(t *testing.T) { - // expectation - errTest := errors.New("test error") - aliasDbMock.On("ListCollectionIDTs", tenantID, ts).Return(nil, errTest).Once() - - // actual - res, gotErr := mockCatalog.ListAliases(ctx, testDb, ts) - require.Nil(t, res) - require.Error(t, gotErr) -} - -func TestTableCatalog_ListAliases_SelectAliasError(t *testing.T) { - // expectation - cidTsPairs := []*dbmodel.CollectionAlias{{CollectionID: collID1, Ts: ts}} - errTest := errors.New("test error") - aliasDbMock.On("ListCollectionIDTs", tenantID, ts).Return(cidTsPairs, nil).Once() - aliasDbMock.On("List", tenantID, mock.Anything).Return(nil, errTest).Once() - - // actual - res, gotErr := mockCatalog.ListAliases(ctx, testDb, ts) - require.Nil(t, res) - require.Error(t, gotErr) -} - -func TestTableCatalog_GetCredential(t *testing.T) { - out := &model.Credential{ - Username: username, - EncryptedPassword: password, - } - user := &dbmodel.User{ - Username: username, - EncryptedPassword: password, - } - - // expectation - userDbMock.On("GetByUsername", tenantID, username).Return(user, nil).Once() - - // actual - res, gotErr := mockCatalog.GetCredential(ctx, username) - require.NoError(t, gotErr) - require.Equal(t, out, res) -} - -func TestTableCatalog_GetCredential_SelectUserError(t *testing.T) { - // expectation - errTest := errors.New("test error") - userDbMock.On("GetByUsername", tenantID, username).Return(nil, errTest).Once() - - // actual - res, gotErr := mockCatalog.GetCredential(ctx, username) - require.Nil(t, res) - require.Error(t, gotErr) -} - -func TestTableCatalog_CreateCredential(t *testing.T) { - in := &model.Credential{ - Username: username, - EncryptedPassword: password, - } - - // expectation - userDbMock.On("Insert", mock.Anything).Return(nil).Once() - - // actual - gotErr := mockCatalog.CreateCredential(ctx, in) - require.NoError(t, gotErr) -} - -func TestTableCatalog_CreateCredential_InsertUserError(t *testing.T) { - in := &model.Credential{ - Username: username, - EncryptedPassword: password, - } - - // expectation - errTest := errors.New("test error") - userDbMock.On("Insert", mock.Anything).Return(errTest).Once() - - // actual - gotErr := mockCatalog.CreateCredential(ctx, in) - require.Error(t, gotErr) -} - -func TestTableCatalog_AlterCredential(t *testing.T) { - in := &model.Credential{ - Username: username, - EncryptedPassword: password, - } - - // expectation - userDbMock.On("UpdatePassword", tenantID, username, password).Return(nil).Once() - - // actual - gotErr := mockCatalog.AlterCredential(ctx, in) - require.NoError(t, gotErr) -} - -func TestTableCatalog_AlterCredential_Error(t *testing.T) { - in := &model.Credential{ - Username: username, - EncryptedPassword: password, - } - - // expectation - errTest := errors.New("test error") - userDbMock.On("UpdatePassword", tenantID, username, password).Return(errTest).Once() - - // actual - gotErr := mockCatalog.AlterCredential(ctx, in) - require.Error(t, gotErr) -} - -func TestTableCatalog_DropCredential(t *testing.T) { - // expectation - userDbMock.On("MarkDeletedByUsername", tenantID, username).Return(nil).Once() - - // actual - gotErr := mockCatalog.DropCredential(ctx, username) - require.NoError(t, gotErr) -} - -func TestTableCatalog_DropCredential_MarkUserDeletedError(t *testing.T) { - // expectation - errTest := errors.New("test error") - userDbMock.On("MarkDeletedByUsername", tenantID, username).Return(errTest).Once() - - // actual - gotErr := mockCatalog.DropCredential(ctx, username) - require.Error(t, gotErr) -} - -func TestTableCatalog_ListCredentials(t *testing.T) { - user := &dbmodel.User{ - Username: username, - EncryptedPassword: password, - } - - // expectation - userDbMock.On("ListUser", tenantID).Return([]*dbmodel.User{user}, nil).Once() - - // actual - res, gotErr := mockCatalog.ListCredentials(ctx) - require.NoError(t, gotErr) - require.Equal(t, []string{username}, res) -} - -func TestTableCatalog_ListCredentials_SelectUsernamesError(t *testing.T) { - // expectation - errTest := errors.New("test error") - userDbMock.On("ListUser", tenantID).Return(nil, errTest).Once() - - // actual - res, gotErr := mockCatalog.ListCredentials(ctx) - require.Nil(t, res) - require.Error(t, gotErr) -} - -func TestTableCatalog_CreateRole(t *testing.T) { - var ( - roleName = "foo" - err error - ) - - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{}, nil).Once() - roleDbMock.On("Insert", mock.Anything).Return(nil).Once() - err = mockCatalog.CreateRole(ctx, tenantID, &milvuspb.RoleEntity{Name: roleName}) - require.NoError(t, err) -} - -func TestTableCatalog_CreateRole_Error(t *testing.T) { - var ( - roleName = "foo" - err error - ) - roleDbMock.On("GetRoles", tenantID, roleName).Return(nil, errors.New("test error")).Once() - err = mockCatalog.CreateRole(ctx, tenantID, &milvuspb.RoleEntity{Name: roleName}) - require.Error(t, err) - - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: 1}}}, nil).Once() - err = mockCatalog.CreateRole(ctx, tenantID, &milvuspb.RoleEntity{Name: roleName}) - require.Equal(t, true, common.IsIgnorableError(err)) - - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{}, nil).Once() - roleDbMock.On("Insert", mock.Anything).Return(errors.New("test error")).Once() - err = mockCatalog.CreateRole(ctx, tenantID, &milvuspb.RoleEntity{Name: roleName}) - require.Error(t, err) -} - -func TestTableCatalog_DropRole(t *testing.T) { - var ( - roleName = "foo" - err error - ) - - roleDbMock.On("Delete", tenantID, roleName).Return(nil).Once() - err = mockCatalog.DropRole(ctx, tenantID, roleName) - require.NoError(t, err) -} - -func TestTableCatalog_DropRole_Error(t *testing.T) { - var ( - roleName = "foo" - err error - ) - - roleDbMock.On("Delete", tenantID, roleName).Return(errors.New("test error")).Once() - err = mockCatalog.DropRole(ctx, tenantID, roleName) - require.Error(t, err) -} - -func TestTableCatalog_AlterUserRole(t *testing.T) { - var ( - username = "foo" - userID = 100 - roleName = "fo" - roleID = 10 - err error - ) - - userDbMock.On("GetByUsername", tenantID, username).Return(&dbmodel.User{ID: int64(userID)}, nil).Once() - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: int64(roleID)}}}, nil).Once() - userRoleDbMock.On("GetUserRoles", tenantID, int64(userID), int64(roleID)).Return([]*dbmodel.UserRole{}, nil).Once() - userRoleDbMock.On("Insert", mock.Anything).Return(nil).Once() - - err = mockCatalog.AlterUserRole(ctx, tenantID, &milvuspb.UserEntity{Name: username}, &milvuspb.RoleEntity{Name: roleName}, milvuspb.OperateUserRoleType_AddUserToRole) - require.NoError(t, err) -} - -func TestTableCatalog_AlterUserRole_GetUserIDError(t *testing.T) { - var ( - username = "foo" - roleName = "fo" - err error - ) - - userDbMock.On("GetByUsername", tenantID, username).Return(nil, errors.New("test error")).Once() - err = mockCatalog.AlterUserRole(ctx, tenantID, &milvuspb.UserEntity{Name: username}, &milvuspb.RoleEntity{Name: roleName}, milvuspb.OperateUserRoleType_AddUserToRole) - require.Error(t, err) -} - -func TestTableCatalog_AlterUserRole_GetRoleIDError(t *testing.T) { - var ( - username = "foo" - userID = 100 - roleName = "fo" - err error - ) - - userDbMock.On("GetByUsername", tenantID, username).Return(&dbmodel.User{ID: int64(userID)}, nil).Once() - roleDbMock.On("GetRoles", tenantID, roleName).Return(nil, errors.New("test error")).Once() - err = mockCatalog.AlterUserRole(ctx, tenantID, &milvuspb.UserEntity{Name: username}, &milvuspb.RoleEntity{Name: roleName}, milvuspb.OperateUserRoleType_AddUserToRole) - require.Error(t, err) -} - -func TestTableCatalog_AlterUserRole_GetUserRoleError(t *testing.T) { - var ( - username = "foo" - userID = 100 - roleName = "fo" - roleID = 10 - err error - ) - - userDbMock.On("GetByUsername", tenantID, username).Return(&dbmodel.User{ID: int64(userID)}, nil).Once() - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: int64(roleID)}}}, nil).Once() - userRoleDbMock.On("GetUserRoles", tenantID, int64(userID), int64(roleID)).Return([]*dbmodel.UserRole{}, errors.New("test error")).Once() - err = mockCatalog.AlterUserRole(ctx, tenantID, &milvuspb.UserEntity{Name: username}, &milvuspb.RoleEntity{Name: roleName}, milvuspb.OperateUserRoleType_AddUserToRole) - require.Error(t, err) -} - -func TestTableCatalog_AlterUserRole_RepeatUserRole(t *testing.T) { - var ( - username = "foo" - userID = 100 - roleName = "fo" - roleID = 10 - err error - ) - - userDbMock.On("GetByUsername", tenantID, username).Return(&dbmodel.User{ID: int64(userID)}, nil).Once() - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: int64(roleID)}}}, nil).Once() - userRoleDbMock.On("GetUserRoles", tenantID, int64(userID), int64(roleID)).Return([]*dbmodel.UserRole{{}}, nil).Once() - err = mockCatalog.AlterUserRole(ctx, tenantID, &milvuspb.UserEntity{Name: username}, &milvuspb.RoleEntity{Name: roleName}, milvuspb.OperateUserRoleType_AddUserToRole) - require.Error(t, err) - require.Equal(t, true, common.IsIgnorableError(err)) -} - -func TestTableCatalog_AlterUserRole_InsertError(t *testing.T) { - var ( - username = "foo" - userID = 100 - roleName = "fo" - roleID = 10 - err error - ) - - userDbMock.On("GetByUsername", tenantID, username).Return(&dbmodel.User{ID: int64(userID)}, nil).Once() - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: int64(roleID)}}}, nil).Once() - userRoleDbMock.On("GetUserRoles", tenantID, int64(userID), int64(roleID)).Return([]*dbmodel.UserRole{}, nil).Once() - userRoleDbMock.On("Insert", mock.Anything).Return(errors.New("test error")).Once() - - err = mockCatalog.AlterUserRole(ctx, tenantID, &milvuspb.UserEntity{Name: username}, &milvuspb.RoleEntity{Name: roleName}, milvuspb.OperateUserRoleType_AddUserToRole) - require.Error(t, err) -} - -func TestTableCatalog_AlterUserRole_Delete(t *testing.T) { - var ( - username = "foo" - userID = 100 - roleName = "fo" - roleID = 10 - err error - ) - - userDbMock.On("GetByUsername", tenantID, username).Return(&dbmodel.User{ID: int64(userID)}, nil).Once() - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: int64(roleID)}}}, nil).Once() - userRoleDbMock.On("GetUserRoles", tenantID, int64(userID), int64(roleID)).Return([]*dbmodel.UserRole{{}}, nil).Once() - userRoleDbMock.On("Delete", tenantID, int64(userID), int64(roleID)).Return(nil).Once() - - err = mockCatalog.AlterUserRole(ctx, tenantID, &milvuspb.UserEntity{Name: username}, &milvuspb.RoleEntity{Name: roleName}, milvuspb.OperateUserRoleType_RemoveUserFromRole) - require.NoError(t, err) -} - -func TestTableCatalog_AlterUserRole_DeleteError(t *testing.T) { - var ( - username = "foo" - userID = 100 - roleName = "fo" - roleID = 10 - err error - ) - - userDbMock.On("GetByUsername", tenantID, username).Return(&dbmodel.User{ID: int64(userID)}, nil).Once() - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: int64(roleID)}}}, nil).Once() - userRoleDbMock.On("GetUserRoles", tenantID, int64(userID), int64(roleID)).Return([]*dbmodel.UserRole{}, nil).Once() - - err = mockCatalog.AlterUserRole(ctx, tenantID, &milvuspb.UserEntity{Name: username}, &milvuspb.RoleEntity{Name: roleName}, milvuspb.OperateUserRoleType_RemoveUserFromRole) - require.Error(t, err) - require.True(t, common.IsIgnorableError(err)) - - userDbMock.On("GetByUsername", tenantID, username).Return(&dbmodel.User{ID: int64(userID)}, nil).Once() - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: int64(roleID)}}}, nil).Once() - userRoleDbMock.On("GetUserRoles", tenantID, int64(userID), int64(roleID)).Return([]*dbmodel.UserRole{{}}, nil).Once() - userRoleDbMock.On("Delete", tenantID, int64(userID), int64(roleID)).Return(errors.New("test error")).Once() - - err = mockCatalog.AlterUserRole(ctx, tenantID, &milvuspb.UserEntity{Name: username}, &milvuspb.RoleEntity{Name: roleName}, milvuspb.OperateUserRoleType_RemoveUserFromRole) - require.Error(t, err) -} - -func TestTableCatalog_AlterUserRole_InvalidType(t *testing.T) { - var ( - username = "foo" - userID = 100 - roleName = "fo" - roleID = 10 - err error - ) - - userDbMock.On("GetByUsername", tenantID, username).Return(&dbmodel.User{ID: int64(userID)}, nil).Once() - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: int64(roleID)}}}, nil).Once() - userRoleDbMock.On("GetUserRoles", tenantID, int64(userID), int64(roleID)).Return([]*dbmodel.UserRole{{}}, nil).Once() - - err = mockCatalog.AlterUserRole(ctx, tenantID, &milvuspb.UserEntity{Name: username}, &milvuspb.RoleEntity{Name: roleName}, 100) - require.Error(t, err) -} - -func TestTableCatalog_ListRole_AllRole(t *testing.T) { - var ( - roleName1 = "foo1" - roleName2 = "foo2" - result []*milvuspb.RoleResult - err error - ) - - roleDbMock.On("GetRoles", tenantID, "").Return([]*dbmodel.Role{{Name: roleName1}, {Name: roleName2}}, nil).Once() - - result, err = mockCatalog.ListRole(ctx, tenantID, nil, false) - require.NoError(t, err) - require.Equal(t, 2, len(result)) - require.Equal(t, roleName1, result[0].Role.Name) -} - -func TestTableCatalog_ListRole_AllRole_IncludeUserInfo(t *testing.T) { - var ( - roleName1 = "foo1" - username1 = "fo1" - username2 = "fo2" - roleName2 = "foo2" - result []*milvuspb.RoleResult - err error - ) - - roleDbMock.On("GetRoles", tenantID, "").Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: 1}, Name: roleName1}, {Base: dbmodel.Base{ID: 10}, Name: roleName2}}, nil).Once() - userRoleDbMock.On("GetUserRoles", tenantID, int64(0), int64(1)).Return([]*dbmodel.UserRole{{User: dbmodel.User{Username: username1}}, {User: dbmodel.User{Username: username2}}}, nil).Once() - userRoleDbMock.On("GetUserRoles", tenantID, int64(0), int64(10)).Return([]*dbmodel.UserRole{}, nil).Once() - - result, err = mockCatalog.ListRole(ctx, tenantID, nil, true) - require.NoError(t, err) - require.Equal(t, 2, len(result)) - require.Equal(t, roleName1, result[0].Role.Name) - require.Equal(t, 2, len(result[0].Users)) - require.Equal(t, username1, result[0].Users[0].Name) - require.Equal(t, username2, result[0].Users[1].Name) - require.Equal(t, roleName2, result[1].Role.Name) -} - -func TestTableCatalog_ListRole_AllRole_Empty(t *testing.T) { - var ( - result []*milvuspb.RoleResult - err error - ) - - roleDbMock.On("GetRoles", tenantID, "").Return([]*dbmodel.Role{}, nil).Once() - - result, err = mockCatalog.ListRole(ctx, tenantID, nil, false) - require.NoError(t, err) - require.Equal(t, 0, len(result)) -} - -func TestTableCatalog_ListRole_OneRole(t *testing.T) { - var ( - roleName1 = "foo1" - result []*milvuspb.RoleResult - err error - ) - - roleDbMock.On("GetRoles", tenantID, roleName1).Return([]*dbmodel.Role{{Name: roleName1}}, nil).Once() - - result, err = mockCatalog.ListRole(ctx, tenantID, &milvuspb.RoleEntity{Name: roleName1}, false) - require.NoError(t, err) - require.Equal(t, 1, len(result)) - require.Equal(t, roleName1, result[0].Role.Name) -} - -func TestTableCatalog_ListRole_OneRole_IncludeUserInfo(t *testing.T) { - var ( - roleName1 = "foo1" - username1 = "fo1" - username2 = "fo2" - result []*milvuspb.RoleResult - err error - ) - - roleDbMock.On("GetRoles", tenantID, roleName1).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: 1}, Name: roleName1}}, nil).Once() - userRoleDbMock.On("GetUserRoles", tenantID, int64(0), int64(1)).Return([]*dbmodel.UserRole{{User: dbmodel.User{Username: username1}}, {User: dbmodel.User{Username: username2}}}, nil).Once() - - result, err = mockCatalog.ListRole(ctx, tenantID, &milvuspb.RoleEntity{Name: roleName1}, true) - require.NoError(t, err) - require.Equal(t, 1, len(result)) - require.Equal(t, roleName1, result[0].Role.Name) - require.Equal(t, 2, len(result[0].Users)) - require.Equal(t, username1, result[0].Users[0].Name) - require.Equal(t, username2, result[0].Users[1].Name) -} - -func TestTableCatalog_ListRole_OneRole_Empty(t *testing.T) { - var ( - roleName = "foo" - err error - ) - - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{}, nil).Once() - - _, err = mockCatalog.ListRole(ctx, tenantID, &milvuspb.RoleEntity{Name: roleName}, false) - require.Error(t, err) -} - -func TestTableCatalog_ListRole_GetRolesError(t *testing.T) { - roleDbMock.On("GetRoles", tenantID, "").Return(nil, errors.New("test error")).Once() - - _, err := mockCatalog.ListRole(ctx, tenantID, nil, false) - require.Error(t, err) -} - -func TestTableCatalog_ListRole_GetUserRolesError(t *testing.T) { - roleDbMock.On("GetRoles", tenantID, "").Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: 1}, Name: "foo"}}, nil).Once() - userRoleDbMock.On("GetUserRoles", tenantID, int64(0), int64(1)).Return(nil, errors.New("test error")).Once() - _, err := mockCatalog.ListRole(ctx, tenantID, nil, true) - require.Error(t, err) -} - -func TestTableCatalog_ListUser_AllUser(t *testing.T) { - var ( - username1 = "foo1" - username2 = "foo2" - result []*milvuspb.UserResult - err error - ) - - userDbMock.On("ListUser", tenantID).Return([]*dbmodel.User{{Username: username1}, {Username: username2}}, nil).Once() - - result, err = mockCatalog.ListUser(ctx, tenantID, nil, false) - require.NoError(t, err) - require.Equal(t, 2, len(result)) - require.Equal(t, username1, result[0].User.Name) -} - -func TestTableCatalog_ListUser_AllUser_IncludeRoleInfo(t *testing.T) { - var ( - roleName1 = "foo1" - username1 = "fo1" - username2 = "fo2" - roleName2 = "foo2" - result []*milvuspb.UserResult - err error - ) - - userDbMock.On("ListUser", tenantID).Return([]*dbmodel.User{{ID: 1, Username: username1}, {ID: 10, Username: username2}}, nil).Once() - userRoleDbMock.On("GetUserRoles", tenantID, int64(1), int64(0)).Return([]*dbmodel.UserRole{{Role: dbmodel.Role{Name: roleName1}}, {Role: dbmodel.Role{Name: roleName2}}}, nil).Once() - userRoleDbMock.On("GetUserRoles", tenantID, int64(10), int64(0)).Return([]*dbmodel.UserRole{}, nil).Once() - - result, err = mockCatalog.ListUser(ctx, tenantID, nil, true) - require.NoError(t, err) - require.Equal(t, 2, len(result)) - require.Equal(t, username1, result[0].User.Name) - require.Equal(t, 2, len(result[0].Roles)) - require.Equal(t, roleName1, result[0].Roles[0].Name) - require.Equal(t, roleName2, result[0].Roles[1].Name) - require.Equal(t, username2, result[1].User.Name) -} - -func TestTableCatalog_ListUser_AllUser_Empty(t *testing.T) { - var ( - result []*milvuspb.UserResult - err error - ) - - userDbMock.On("ListUser", tenantID).Return([]*dbmodel.User{}, nil).Once() - - result, err = mockCatalog.ListUser(ctx, tenantID, nil, false) - require.NoError(t, err) - require.Equal(t, 0, len(result)) -} - -func TestTableCatalog_ListUser_OneUser(t *testing.T) { - var ( - username1 = "foo1" - result []*milvuspb.UserResult - err error - ) - - userDbMock.On("GetByUsername", tenantID, username1).Return(&dbmodel.User{Username: username1}, nil).Once() - - result, err = mockCatalog.ListUser(ctx, tenantID, &milvuspb.UserEntity{Name: username1}, false) - require.NoError(t, err) - require.Equal(t, 1, len(result)) - require.Equal(t, username1, result[0].User.Name) -} - -func TestTableCatalog_ListUser_OneUser_IncludeRoleInfo(t *testing.T) { - var ( - roleName1 = "foo1" - roleName2 = "foo1" - username1 = "fo1" - result []*milvuspb.UserResult - err error - ) - - userDbMock.On("GetByUsername", tenantID, username1).Return(&dbmodel.User{ID: 1, Username: username1}, nil).Once() - userRoleDbMock.On("GetUserRoles", tenantID, int64(1), int64(0)). - Return([]*dbmodel.UserRole{{Role: dbmodel.Role{Name: roleName1}}, {Role: dbmodel.Role{Name: roleName2}}}, nil).Once() - - result, err = mockCatalog.ListUser(ctx, tenantID, &milvuspb.UserEntity{Name: username1}, true) - require.NoError(t, err) - require.Equal(t, 1, len(result)) - require.Equal(t, username1, result[0].User.Name) - require.Equal(t, 2, len(result[0].Roles)) - require.Equal(t, roleName1, result[0].Roles[0].Name) - require.Equal(t, roleName2, result[0].Roles[1].Name) -} - -func TestTableCatalog_ListUser_ListUserError(t *testing.T) { - userDbMock.On("ListUser", tenantID).Return(nil, errors.New("test error")).Once() - - _, err := mockCatalog.ListUser(ctx, tenantID, nil, false) - require.Error(t, err) -} - -func TestTableCatalog_ListUser_GetByUsernameError(t *testing.T) { - var ( - username1 = "foo" - err error - ) - - userDbMock.On("GetByUsername", tenantID, username1).Return(nil, errors.New("test error")).Once() - - _, err = mockCatalog.ListUser(ctx, tenantID, &milvuspb.UserEntity{Name: username1}, false) - require.Error(t, err) -} - -func TestTableCatalog_ListUser_GetUserRolesError(t *testing.T) { - var ( - username1 = "foo" - err error - ) - - userDbMock.On("GetByUsername", tenantID, username1).Return(&dbmodel.User{ID: 1, Username: username1}, nil).Once() - userRoleDbMock.On("GetUserRoles", tenantID, int64(1), int64(0)).Return(nil, errors.New("test error")).Once() - - _, err = mockCatalog.ListUser(ctx, tenantID, &milvuspb.UserEntity{Name: username1}, true) - require.Error(t, err) -} - -func TestTableCatalog_AlterGrant_Revoke(t *testing.T) { - var ( - roleName = "foo" - roleID int64 = 1 - object = "Collection" - objectName = "col1" - grantID int64 = 10 - username = "fo" - privilege = "PrivilegeLoad" - grant *milvuspb.GrantEntity - err error - ) - grant = &milvuspb.GrantEntity{ - Role: &milvuspb.RoleEntity{Name: roleName}, - Object: &milvuspb.ObjectEntity{Name: object}, - ObjectName: objectName, - Grantor: &milvuspb.GrantorEntity{ - User: &milvuspb.UserEntity{Name: username}, - Privilege: &milvuspb.PrivilegeEntity{Name: privilege}, - }, - } - - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: roleID}}}, nil).Once() - grantDbMock.On("GetGrants", tenantID, roleID, object, objectName).Return([]*dbmodel.Grant{{Base: dbmodel.Base{ID: grantID}}}, nil).Once() - grantIDDbMock.On("GetGrantIDs", tenantID, grantID, privilege, mock.Anything, mock.Anything).Return([]*dbmodel.GrantID{{}}, nil).Once() - grantIDDbMock.On("Delete", tenantID, grantID, privilege).Return(nil).Once() - err = mockCatalog.AlterGrant(ctx, tenantID, grant, milvuspb.OperatePrivilegeType_Revoke) - require.NoError(t, err) - - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: roleID}}}, nil).Once() - grantDbMock.On("GetGrants", tenantID, roleID, object, objectName).Return(nil, errors.New("test error")).Once() - err = mockCatalog.AlterGrant(ctx, tenantID, grant, milvuspb.OperatePrivilegeType_Revoke) - require.Error(t, err) - - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: roleID}}}, nil).Once() - grantDbMock.On("GetGrants", tenantID, roleID, object, objectName).Return(nil, nil).Once() - err = mockCatalog.AlterGrant(ctx, tenantID, grant, milvuspb.OperatePrivilegeType_Revoke) - require.Error(t, err) - require.True(t, common.IsIgnorableError(err)) - - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: roleID}}}, nil).Once() - grantDbMock.On("GetGrants", tenantID, roleID, object, objectName).Return([]*dbmodel.Grant{{Base: dbmodel.Base{ID: grantID}}}, nil).Once() - grantIDDbMock.On("GetGrantIDs", tenantID, grantID, privilege, mock.Anything, mock.Anything).Return(nil, errors.New("test error")).Once() - err = mockCatalog.AlterGrant(ctx, tenantID, grant, milvuspb.OperatePrivilegeType_Revoke) - require.Error(t, err) - - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: roleID}}}, nil).Once() - grantDbMock.On("GetGrants", tenantID, roleID, object, objectName).Return([]*dbmodel.Grant{{Base: dbmodel.Base{ID: grantID}}}, nil).Once() - grantIDDbMock.On("GetGrantIDs", tenantID, grantID, privilege, mock.Anything, mock.Anything).Return(nil, nil).Once() - err = mockCatalog.AlterGrant(ctx, tenantID, grant, milvuspb.OperatePrivilegeType_Revoke) - require.Error(t, err) - require.True(t, common.IsIgnorableError(err)) - - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: roleID}}}, nil).Once() - grantDbMock.On("GetGrants", tenantID, roleID, object, objectName).Return([]*dbmodel.Grant{{Base: dbmodel.Base{ID: grantID}}}, nil).Once() - grantIDDbMock.On("GetGrantIDs", tenantID, grantID, privilege, mock.Anything, mock.Anything).Return([]*dbmodel.GrantID{{}}, nil).Once() - grantIDDbMock.On("Delete", tenantID, grantID, privilege).Return(errors.New("test error")).Once() - err = mockCatalog.AlterGrant(ctx, tenantID, grant, milvuspb.OperatePrivilegeType_Revoke) - require.Error(t, err) -} - -func TestTableCatalog_AlterGrant_Grant(t *testing.T) { - var ( - roleName = "foo" - roleID int64 = 1 - object = "Collection" - objectName = "col1" - grantID int64 = 10 - username = "fo" - userID int64 = 100 - privilege = "PrivilegeLoad" - grant *milvuspb.GrantEntity - err error - ) - grant = &milvuspb.GrantEntity{ - Role: &milvuspb.RoleEntity{Name: roleName}, - Object: &milvuspb.ObjectEntity{Name: object}, - ObjectName: objectName, - Grantor: &milvuspb.GrantorEntity{ - User: &milvuspb.UserEntity{Name: username}, - Privilege: &milvuspb.PrivilegeEntity{Name: privilege}, - }, - } - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: roleID}}}, nil).Once() - grantDbMock.On("GetGrants", tenantID, roleID, object, objectName).Return([]*dbmodel.Grant{{Base: dbmodel.Base{ID: grantID}}}, nil).Once() - grantIDDbMock.On("GetGrantIDs", tenantID, grantID, privilege, mock.Anything, mock.Anything).Return([]*dbmodel.GrantID{}, nil).Once() - userDbMock.On("GetByUsername", tenantID, username).Return(&dbmodel.User{ID: userID}, nil).Once() - grantIDDbMock.On("Insert", mock.Anything).Return(nil).Once() - err = mockCatalog.AlterGrant(ctx, tenantID, grant, milvuspb.OperatePrivilegeType_Grant) - require.NoError(t, err) - - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: roleID}}}, nil).Once() - grantDbMock.On("GetGrants", tenantID, roleID, object, objectName).Return(nil, nil).Once() - grantDbMock.On("Insert", mock.Anything).Return(nil).Once() - grantIDDbMock.On("GetGrantIDs", tenantID, mock.Anything, privilege, mock.Anything, mock.Anything).Return([]*dbmodel.GrantID{}, nil).Once() - userDbMock.On("GetByUsername", tenantID, username).Return(&dbmodel.User{ID: userID}, nil).Once() - grantIDDbMock.On("Insert", mock.Anything).Return(nil).Once() - err = mockCatalog.AlterGrant(ctx, tenantID, grant, milvuspb.OperatePrivilegeType_Grant) - require.NoError(t, err) - - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: roleID}}}, nil).Once() - grantDbMock.On("GetGrants", tenantID, roleID, object, objectName).Return([]*dbmodel.Grant{{Base: dbmodel.Base{ID: grantID}}}, nil).Once() - grantIDDbMock.On("GetGrantIDs", tenantID, grantID, privilege, mock.Anything, mock.Anything).Return(nil, errors.New("test error")).Once() - err = mockCatalog.AlterGrant(ctx, tenantID, grant, milvuspb.OperatePrivilegeType_Grant) - require.Error(t, err) - - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: roleID}}}, nil).Once() - grantDbMock.On("GetGrants", tenantID, roleID, object, objectName).Return([]*dbmodel.Grant{{Base: dbmodel.Base{ID: grantID}}}, nil).Once() - grantIDDbMock.On("GetGrantIDs", tenantID, grantID, privilege, mock.Anything, mock.Anything).Return([]*dbmodel.GrantID{{}}, nil).Once() - err = mockCatalog.AlterGrant(ctx, tenantID, grant, milvuspb.OperatePrivilegeType_Grant) - require.Error(t, err) - require.True(t, common.IsIgnorableError(err)) - - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: roleID}}}, nil).Once() - grantDbMock.On("GetGrants", tenantID, roleID, object, objectName).Return([]*dbmodel.Grant{{Base: dbmodel.Base{ID: grantID}}}, nil).Once() - grantIDDbMock.On("GetGrantIDs", tenantID, grantID, privilege, mock.Anything, mock.Anything).Return([]*dbmodel.GrantID{}, nil).Once() - userDbMock.On("GetByUsername", tenantID, username).Return(nil, errors.New("test error")).Once() - err = mockCatalog.AlterGrant(ctx, tenantID, grant, milvuspb.OperatePrivilegeType_Grant) - require.Error(t, err) - - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: roleID}}}, nil).Once() - grantDbMock.On("GetGrants", tenantID, roleID, object, objectName).Return([]*dbmodel.Grant{{Base: dbmodel.Base{ID: grantID}}}, nil).Once() - grantIDDbMock.On("GetGrantIDs", tenantID, grantID, privilege, mock.Anything, mock.Anything).Return([]*dbmodel.GrantID{}, nil).Once() - userDbMock.On("GetByUsername", tenantID, username).Return(&dbmodel.User{ID: userID}, nil).Once() - grantIDDbMock.On("Insert", mock.Anything).Return(errors.New("test error")).Once() - err = mockCatalog.AlterGrant(ctx, tenantID, grant, milvuspb.OperatePrivilegeType_Grant) - require.Error(t, err) -} - -func TestTableCatalog_AlterGrant_InvalidType(t *testing.T) { - var ( - roleName = "foo" - err error - ) - - err = mockCatalog.AlterGrant(ctx, tenantID, &milvuspb.GrantEntity{Role: &milvuspb.RoleEntity{Name: roleName}}, 100) - require.Error(t, err) -} - -func TestTableCatalog_ListGrant(t *testing.T) { - var ( - roleID int64 = 1 - grantID int64 = 10 - grant *milvuspb.GrantEntity - grants []*dbmodel.Grant - entites []*milvuspb.GrantEntity - err error - ) - - grant = &milvuspb.GrantEntity{ - Role: &milvuspb.RoleEntity{Name: "foo"}, - Object: &milvuspb.ObjectEntity{Name: "Collection"}, - ObjectName: "col1", - } - grants = []*dbmodel.Grant{ - { - Base: dbmodel.Base{ID: grantID}, - Role: dbmodel.Role{Name: "foo"}, - Object: "Collection", - ObjectName: "col1", - }, - } - roleDbMock.On("GetRoles", tenantID, grant.Role.Name).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: roleID}}}, nil).Once() - grantDbMock.On("GetGrants", tenantID, roleID, grant.Object.Name, grant.ObjectName).Return(grants, nil).Once() - grantIDDbMock.On("GetGrantIDs", tenantID, grantID, "", false, true).Return([]*dbmodel.GrantID{ - { - Privilege: "PrivilegeLoad", - Grantor: dbmodel.User{Username: "root"}, - }, - { - Privilege: "*", - Grantor: dbmodel.User{Username: "root"}, - }, - }, nil).Once() - - entites, err = mockCatalog.ListGrant(ctx, tenantID, grant) - require.NoError(t, err) - require.Equal(t, 2, len(entites)) - require.Equal(t, "foo", entites[0].Role.Name) - require.Equal(t, "Collection", entites[1].Object.Name) - require.Equal(t, "col1", entites[1].ObjectName) - require.Equal(t, "root", entites[1].Grantor.User.Name) - require.Equal(t, "*", entites[1].Grantor.Privilege.Name) -} - -func TestTableCatalog_ListGrant_GetRolesError(t *testing.T) { - var ( - grant *milvuspb.GrantEntity - err error - ) - grant = &milvuspb.GrantEntity{ - Role: &milvuspb.RoleEntity{Name: "foo"}, - Object: &milvuspb.ObjectEntity{Name: "Collection"}, - ObjectName: "col1", - } - roleDbMock.On("GetRoles", tenantID, grant.Role.Name).Return(nil, errors.New("test error")).Once() - _, err = mockCatalog.ListGrant(ctx, tenantID, grant) - require.Error(t, err) -} - -func TestTableCatalog_ListGrant_GetGrantError(t *testing.T) { - var ( - grant *milvuspb.GrantEntity - err error - ) - grant = &milvuspb.GrantEntity{ - Role: &milvuspb.RoleEntity{Name: "foo"}, - Object: &milvuspb.ObjectEntity{Name: "Collection"}, - ObjectName: "col1", - } - roleDbMock.On("GetRoles", tenantID, grant.Role.Name).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: 1}}}, nil).Once() - grantDbMock.On("GetGrants", tenantID, int64(1), grant.Object.Name, grant.ObjectName).Return(nil, errors.New("test error")).Once() - _, err = mockCatalog.ListGrant(ctx, tenantID, grant) - require.Error(t, err) -} - -func TestTableCatalog_ListGrant_GetGrantIDError(t *testing.T) { - var ( - roleID int64 = 1 - grantID int64 = 10 - grant *milvuspb.GrantEntity - grants []*dbmodel.Grant - err error - ) - - grant = &milvuspb.GrantEntity{ - Role: &milvuspb.RoleEntity{Name: "foo"}, - Object: &milvuspb.ObjectEntity{Name: "Collection"}, - ObjectName: "col1", - } - grants = []*dbmodel.Grant{ - { - Base: dbmodel.Base{ID: grantID}, - Role: dbmodel.Role{Name: "foo"}, - Object: "Collection", - ObjectName: "col1", - }, - } - roleDbMock.On("GetRoles", tenantID, grant.Role.Name).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: roleID}}}, nil).Once() - grantDbMock.On("GetGrants", tenantID, roleID, grant.Object.Name, grant.ObjectName).Return(grants, nil).Once() - grantIDDbMock.On("GetGrantIDs", tenantID, grantID, "", false, true).Return(nil, errors.New("test error")).Once() - - _, err = mockCatalog.ListGrant(ctx, tenantID, grant) - require.Error(t, err) -} - -func TestTableCatalog_ListGrant_NotExistError(t *testing.T) { - var ( - grant *milvuspb.GrantEntity - err error - ) - grant = &milvuspb.GrantEntity{ - Role: &milvuspb.RoleEntity{Name: "foo"}, - Object: &milvuspb.ObjectEntity{Name: "Collection"}, - ObjectName: "col1", - } - roleDbMock.On("GetRoles", tenantID, grant.Role.Name).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: 1}}}, nil).Once() - grantDbMock.On("GetGrants", tenantID, int64(1), grant.Object.Name, grant.ObjectName).Return(nil, nil).Once() - - _, err = mockCatalog.ListGrant(ctx, tenantID, grant) - require.Error(t, err) - require.True(t, common.IsKeyNotExistError(err)) -} - -func TestTableCatalog_DropGrant(t *testing.T) { - var ( - roleName = "foo" - roleID int64 = 10 - err error - ) - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: roleID}}}, nil).Once() - grantDbMock.On("Delete", tenantID, roleID, "", "").Return(nil).Once() - err = mockCatalog.DeleteGrant(ctx, tenantID, &milvuspb.RoleEntity{Name: roleName}) - require.NoError(t, err) - - roleDbMock.On("GetRoles", tenantID, roleName).Return(nil, errors.New("test error")).Once() - err = mockCatalog.DeleteGrant(ctx, tenantID, &milvuspb.RoleEntity{Name: roleName}) - require.Error(t, err) - - roleDbMock.On("GetRoles", tenantID, roleName).Return([]*dbmodel.Role{{Base: dbmodel.Base{ID: roleID}}}, nil).Once() - grantDbMock.On("Delete", tenantID, roleID, "", "").Return(errors.New("test error")).Once() - err = mockCatalog.DeleteGrant(ctx, tenantID, &milvuspb.RoleEntity{Name: roleName}) - require.Error(t, err) -} - -func TestTableCatalog_ListPolicy(t *testing.T) { - var ( - roleName1 = "foo1" - roleName2 = "foo1" - grantID1 int64 = 10 - grantID2 int64 = 100 - object1 = "obj1" - object2 = "obj2" - objectName1 = "col1" - objectName2 = "col2" - privilege1 = "PrivilegeInsert" - privilege2 = "PrivilegeQuery" - grants []*dbmodel.Grant - policies []string - err error - ) - - grants = []*dbmodel.Grant{ - { - Base: dbmodel.Base{ID: grantID1}, - Role: dbmodel.Role{Name: roleName1}, - Object: object1, - ObjectName: objectName1, - }, - { - Base: dbmodel.Base{ID: grantID2}, - Role: dbmodel.Role{Name: roleName2}, - Object: object2, - ObjectName: objectName2, - }, - } - grantDbMock.On("GetGrants", tenantID, int64(0), "", "").Return(grants, nil).Once() - grantIDDbMock.On("GetGrantIDs", tenantID, grantID1, "", false, false).Return([]*dbmodel.GrantID{ - {Privilege: privilege1}, - {Privilege: privilege2}, - }, nil).Once() - grantIDDbMock.On("GetGrantIDs", tenantID, grantID2, "", false, false).Return([]*dbmodel.GrantID{ - {Privilege: privilege1}, - }, nil).Once() - - policies, err = mockCatalog.ListPolicy(ctx, tenantID) - require.NoError(t, err) - require.Equal(t, 3, len(policies)) - require.Equal(t, funcutil.PolicyForPrivilege(roleName1, object1, objectName1, privilege1, util.DefaultDBName), policies[0]) - - grantDbMock.On("GetGrants", tenantID, int64(0), "", "").Return(nil, errors.New("test error")).Once() - _, err = mockCatalog.ListPolicy(ctx, tenantID) - require.Error(t, err) - - grantDbMock.On("GetGrants", tenantID, int64(0), "", "").Return(grants, nil).Once() - grantIDDbMock.On("GetGrantIDs", tenantID, grantID1, "", false, false).Return(nil, errors.New("test error")).Once() - _, err = mockCatalog.ListPolicy(ctx, tenantID) - require.Error(t, err) -} - -func TestTableCatalog_ListUserRole(t *testing.T) { - var ( - username1 = "foo1" - username2 = "foo2" - roleName1 = "fo1" - roleName2 = "fo2" - userRoles []string - err error - ) - - userRoleDbMock.On("GetUserRoles", tenantID, int64(0), int64(0)).Return([]*dbmodel.UserRole{ - {User: dbmodel.User{Username: username1}, Role: dbmodel.Role{Name: roleName1}}, - {User: dbmodel.User{Username: username2}, Role: dbmodel.Role{Name: roleName2}}, - }, nil).Once() - - userRoles, err = mockCatalog.ListUserRole(ctx, tenantID) - require.NoError(t, err) - require.Equal(t, 2, len(userRoles)) - require.Equal(t, funcutil.EncodeUserRoleCache(username1, roleName1), userRoles[0]) -} - -func TestTableCatalog_ListUserRole_Error(t *testing.T) { - userRoleDbMock.On("GetUserRoles", tenantID, int64(0), int64(0)).Return(nil, errors.New("test error")).Once() - _, err := mockCatalog.ListUserRole(ctx, tenantID) - require.Error(t, err) -} diff --git a/internal/rootcoord/root_coord.go b/internal/rootcoord/root_coord.go index 9e1f740ba1..53fabac3de 100644 --- a/internal/rootcoord/root_coord.go +++ b/internal/rootcoord/root_coord.go @@ -39,9 +39,6 @@ import ( "github.com/milvus-io/milvus/internal/kv" etcdkv "github.com/milvus-io/milvus/internal/kv/etcd" "github.com/milvus-io/milvus/internal/metastore" - "github.com/milvus-io/milvus/internal/metastore/db/dao" - "github.com/milvus-io/milvus/internal/metastore/db/dbcore" - "github.com/milvus-io/milvus/internal/metastore/db/rootcoord" kvmetestore "github.com/milvus-io/milvus/internal/metastore/kv/rootcoord" "github.com/milvus-io/milvus/internal/metastore/model" pb "github.com/milvus-io/milvus/internal/proto/etcdpb" @@ -344,14 +341,6 @@ func (c *Core) initMetaTable() error { } catalog = &kvmetestore.Catalog{Txn: metaKV, Snapshot: ss} - case util.MetaStoreTypeMysql: - // connect to database - err := dbcore.Connect(&Params.DBCfg) - if err != nil { - return err - } - - catalog = rootcoord.NewTableCatalog(dbcore.NewTxImpl(), dao.NewMetaDomain()) default: return retry.Unrecoverable(fmt.Errorf("not supported meta store: %s", Params.MetaStoreCfg.MetaStoreType.GetValue())) } diff --git a/pkg/go.sum b/pkg/go.sum index 7e9d5fc7df..3ed3a90b8a 100644 --- a/pkg/go.sum +++ b/pkg/go.sum @@ -211,7 +211,6 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre github.com/go-martini/martini v0.0.0-20170121215854-22fa46961aab/go.mod h1:/P9AEU963A2AYjv4d1V5eVL1CQbEJq6aCNHDDjibzu8= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= -github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= diff --git a/pkg/util/constant.go b/pkg/util/constant.go index 22025da449..6c11be3f34 100644 --- a/pkg/util/constant.go +++ b/pkg/util/constant.go @@ -26,7 +26,6 @@ import ( // Meta Prefix consts const ( MetaStoreTypeEtcd = "etcd" - MetaStoreTypeMysql = "mysql" SegmentMetaPrefix = "queryCoord-segmentMeta" ChangeInfoMetaPrefix = "queryCoord-sealedSegmentChangeInfo" diff --git a/pkg/util/paramtable/base_table.go b/pkg/util/paramtable/base_table.go index ed8db6a6b5..c329deafc6 100644 --- a/pkg/util/paramtable/base_table.go +++ b/pkg/util/paramtable/base_table.go @@ -55,7 +55,7 @@ const ( // Const of Global Config List func globalConfigPrefixs() []string { - return []string{"metastore", "localStorage", "etcd", "mysql", "minio", "pulsar", "kafka", "rocksmq", "log", "grpc", "common", "quotaAndLimits"} + return []string{"metastore", "localStorage", "etcd", "minio", "pulsar", "kafka", "rocksmq", "log", "grpc", "common", "quotaAndLimits"} } var defaultYaml = []string{"milvus.yaml", "default.yaml", "user.yaml"} diff --git a/pkg/util/paramtable/service_param.go b/pkg/util/paramtable/service_param.go index e55c6e2e1b..1b6a7bd717 100644 --- a/pkg/util/paramtable/service_param.go +++ b/pkg/util/paramtable/service_param.go @@ -47,7 +47,6 @@ type ServiceParam struct { LocalStorageCfg LocalStorageConfig MetaStoreCfg MetaStoreConfig EtcdCfg EtcdConfig - DBCfg MetaDBConfig MQCfg MQConfig PulsarCfg PulsarConfig KafkaCfg KafkaConfig @@ -62,7 +61,6 @@ func (p *ServiceParam) init() { p.LocalStorageCfg.Init(&p.BaseTable) p.MetaStoreCfg.Init(&p.BaseTable) p.EtcdCfg.Init(&p.BaseTable) - p.DBCfg.Init(&p.BaseTable) p.MQCfg.Init(&p.BaseTable) p.PulsarCfg.Init(&p.BaseTable) p.KafkaCfg.Init(&p.BaseTable) @@ -287,83 +285,12 @@ func (p *MetaStoreConfig) Init(base *BaseTable) { Key: "metastore.type", Version: "2.2.0", DefaultValue: util.MetaStoreTypeEtcd, - Doc: `Default value: etcd -Valid values: [etcd, mysql]`, + Doc: `Default value: etcd, Valid values: etcd `, Export: true, } p.MetaStoreType.Init(base.mgr) } -// ///////////////////////////////////////////////////////////////////////////// -// --- meta db --- -type MetaDBConfig struct { - Username ParamItem `refreshable:"false"` - Password ParamItem `refreshable:"false"` - Address ParamItem `refreshable:"false"` - Port ParamItem `refreshable:"false"` - DBName ParamItem `refreshable:"false"` - MaxOpenConns ParamItem `refreshable:"false"` - MaxIdleConns ParamItem `refreshable:"false"` -} - -func (p *MetaDBConfig) Init(base *BaseTable) { - p.Username = ParamItem{ - Key: "mysql.username", - Version: "2.2.0", - PanicIfEmpty: true, - Export: true, - } - p.Username.Init(base.mgr) - - p.Password = ParamItem{ - Key: "mysql.password", - Version: "2.2.0", - PanicIfEmpty: true, - Export: true, - } - p.Password.Init(base.mgr) - - p.Address = ParamItem{ - Key: "mysql.address", - Version: "2.2.0", - PanicIfEmpty: true, - Export: true, - } - p.Address.Init(base.mgr) - - p.Port = ParamItem{ - Key: "mysql.port", - Version: "2.2.0", - DefaultValue: "3306", - Export: true, - } - p.Port.Init(base.mgr) - - p.DBName = ParamItem{ - Key: "mysql.dbName", - Version: "2.2.0", - PanicIfEmpty: true, - Export: true, - } - p.DBName.Init(base.mgr) - - p.MaxOpenConns = ParamItem{ - Key: "mysql.maxOpenConns", - Version: "2.2.0", - DefaultValue: "20", - Export: true, - } - p.MaxOpenConns.Init(base.mgr) - - p.MaxIdleConns = ParamItem{ - Key: "mysql.maxIdleConns", - Version: "2.2.0", - DefaultValue: "5", - Export: true, - } - p.MaxIdleConns.Init(base.mgr) -} - // ///////////////////////////////////////////////////////////////////////////// // --- mq --- diff --git a/tests/benchmark/milvus_benchmark/env/helm_utils.py b/tests/benchmark/milvus_benchmark/env/helm_utils.py index 2d1caec8ec..2e2985c510 100644 --- a/tests/benchmark/milvus_benchmark/env/helm_utils.py +++ b/tests/benchmark/milvus_benchmark/env/helm_utils.py @@ -151,25 +151,7 @@ def update_values(file_path, deploy_mode, hostname, server_tag, milvus_config, s # if "replicas" in milvus_config["readonly"]: # values_dict["readonly"]["replicas"] = milvus_config["readonly"]["replicas"] - # use_external_mysql = False - # if "external_mysql" in milvus_config and milvus_config["external_mysql"]: - # use_external_mysql = True - # # meta mysql - # if use_external_mysql: - # values_dict["mysql"]["enabled"] = False - # # values_dict["mysql"]["persistence"]["enabled"] = True - # # values_dict["mysql"]["persistence"]["existingClaim"] = hashlib.md5(path_value.encode(encoding='UTF-8')).hexdigest() - # values_dict['externalMysql']['enabled'] = True - # if deploy_mode == "local": - # values_dict['externalMysql']["ip"] = "192.168.1.238" - # else: - # values_dict['externalMysql']["ip"] = "milvus-mysql.test" - # values_dict['externalMysql']["port"] = 3306 - # values_dict['externalMysql']["user"] = "root" - # values_dict['externalMysql']["password"] = "milvus" - # values_dict['externalMysql']["database"] = "db" - # else: - # values_dict["mysql"]["enabled"] = False + # # update values.yaml with the given host node_config = None perf_tolerations = [{ @@ -351,7 +333,7 @@ def restart_server(helm_release_name, namespace): # body = {"replicas": 0} pods = v1.list_namespaced_pod(namespace) for i in pods.items: - if i.metadata.name.find(helm_release_name) != -1 and i.metadata.name.find("mysql") == -1: + if i.metadata.name.find(helm_release_name) != -1: pod_name = i.metadata.name break # v1.patch_namespaced_config_map(config_map_name, namespace, body, pretty='true') @@ -374,7 +356,7 @@ def restart_server(helm_release_name, namespace): logger.error(pod_name_tmp) if pod_name_tmp == pod_name: continue - elif pod_name_tmp.find(helm_release_name) == -1 or pod_name_tmp.find("mysql") != -1: + elif pod_name_tmp.find(helm_release_name) == -1: continue else: status_res = v1.read_namespaced_pod_status(pod_name_tmp, namespace, pretty='true') diff --git a/tests/benchmark/milvus_benchmark/suites/011_cluster_cpu_accuracy_ann.yaml b/tests/benchmark/milvus_benchmark/suites/011_cluster_cpu_accuracy_ann.yaml index 63f26c1307..30fd7678e8 100644 --- a/tests/benchmark/milvus_benchmark/suites/011_cluster_cpu_accuracy_ann.yaml +++ b/tests/benchmark/milvus_benchmark/suites/011_cluster_cpu_accuracy_ann.yaml @@ -18,7 +18,7 @@ ann_accuracy: cluster: true readonly: replicas: 2 - external_mysql: true + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 collection_name: sift_128_euclidean index_types: ['flat'] @@ -47,7 +47,7 @@ ann_accuracy: cluster: true readonly: replicas: 2 - external_mysql: true + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 collection_name: sift_128_euclidean @@ -77,7 +77,7 @@ ann_accuracy: cluster: true readonly: replicas: 2 - external_mysql: true + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 collection_name: sift_128_euclidean @@ -108,7 +108,7 @@ ann_accuracy: cluster: true readonly: replicas: 2 - external_mysql: true + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 collection_name: sift_128_euclidean @@ -138,7 +138,7 @@ ann_accuracy: cluster: true readonly: replicas: 2 - external_mysql: true + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 collection_name: sift_128_euclidean @@ -169,7 +169,7 @@ ann_accuracy: cluster: true readonly: replicas: 2 - external_mysql: true + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 collection_name: glove_200_angular @@ -199,7 +199,7 @@ ann_accuracy: cluster: true readonly: replicas: 2 - external_mysql: true + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 collection_name: glove_200_angular @@ -229,7 +229,7 @@ ann_accuracy: cluster: true readonly: replicas: 2 - external_mysql: true + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 collection_name: glove_200_angular @@ -259,7 +259,7 @@ ann_accuracy: cluster: true readonly: replicas: 2 - external_mysql: true + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 collection_name: glove_200_angular @@ -289,7 +289,7 @@ ann_accuracy: cluster: true readonly: replicas: 2 - external_mysql: true + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 collection_name: glove_200_angular @@ -320,7 +320,7 @@ ann_accuracy: cluster: true readonly: replicas: 2 - external_mysql: true + source_file: /test/milvus/ann_hdf5/glove-200-angular.hdf5 collection_name: glove_200_angular diff --git a/tests/benchmark/milvus_benchmark/suites/011_cpu_accuracy_ann_debug.yaml b/tests/benchmark/milvus_benchmark/suites/011_cpu_accuracy_ann_debug.yaml index 59fdf1f74c..6e967f4b59 100644 --- a/tests/benchmark/milvus_benchmark/suites/011_cpu_accuracy_ann_debug.yaml +++ b/tests/benchmark/milvus_benchmark/suites/011_cpu_accuracy_ann_debug.yaml @@ -15,7 +15,7 @@ ann_accuracy: cluster: true readonly: replicas: 2 - external_mysql: true + source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 collection_name: sift_128_euclidean index_types: ['ivf_flat', 'ivf_sq8'] @@ -38,7 +38,6 @@ ann_accuracy: # gpu_resource_config.build_index_resources: # - gpu0 # cluster: false - # external_mysql: false # source_file: /test/milvus/ann_hdf5/sift-128-euclidean.hdf5 # collection_name: sift_128_euclidean # index_types: ['ivf_flat', 'ivf_sq8'] diff --git a/tests/benchmark/milvus_benchmark/suites/011_cpu_search_sift10m.yaml b/tests/benchmark/milvus_benchmark/suites/011_cpu_search_sift10m.yaml index 6e12b3a61d..8bee1234d2 100644 --- a/tests/benchmark/milvus_benchmark/suites/011_cpu_search_sift10m.yaml +++ b/tests/benchmark/milvus_benchmark/suites/011_cpu_search_sift10m.yaml @@ -39,7 +39,7 @@ search_performance: cluster: true readonly: replicas: 2 - external_mysql: true + collection_name: sift_10m_128_l2_011 run_count: 2 top_ks: [1, 10, 100, 1000] diff --git a/tests/benchmark/milvus_benchmark/suites/011_gpu_search_debug.yaml b/tests/benchmark/milvus_benchmark/suites/011_gpu_search_debug.yaml index fa723a6aef..78c575d5e1 100644 --- a/tests/benchmark/milvus_benchmark/suites/011_gpu_search_debug.yaml +++ b/tests/benchmark/milvus_benchmark/suites/011_gpu_search_debug.yaml @@ -66,7 +66,7 @@ search_performance: # - gpu1 # wal_enable: true # cluster: true - # external_mysql: true + # # collection_name: sift_50m_128_l2_011 # run_count: 2 # top_ks: [1, 10, 100, 1000] diff --git a/tests/benchmark/milvus_benchmark/suites/011_insert_debug.yaml b/tests/benchmark/milvus_benchmark/suites/011_insert_debug.yaml index 34a598dabe..e64868e9a9 100644 --- a/tests/benchmark/milvus_benchmark/suites/011_insert_debug.yaml +++ b/tests/benchmark/milvus_benchmark/suites/011_insert_debug.yaml @@ -15,7 +15,7 @@ insert_performance: - gpu0 - gpu1 wal_enable: true - external_mysql: true + cluster: true collection_name: sift_1b_128_l2_sq8_011 ni_per: 50000 diff --git a/tests/benchmark/milvus_benchmark/suites/cluster_locust_mix.yaml b/tests/benchmark/milvus_benchmark/suites/cluster_locust_mix.yaml index cb1b457644..9994a4a0a2 100644 --- a/tests/benchmark/milvus_benchmark/suites/cluster_locust_mix.yaml +++ b/tests/benchmark/milvus_benchmark/suites/cluster_locust_mix.yaml @@ -14,7 +14,7 @@ locust_mix_performance: gpu_resource_config.build_index_resources: gpu0 wal_enable: true - external_mysql: true + cluster: true readonly: replicas: 2 diff --git a/tests/benchmark/milvus_benchmark/suites/debug_gpu_search.yaml b/tests/benchmark/milvus_benchmark/suites/debug_gpu_search.yaml index 924d7fb684..24a871a40c 100644 --- a/tests/benchmark/milvus_benchmark/suites/debug_gpu_search.yaml +++ b/tests/benchmark/milvus_benchmark/suites/debug_gpu_search.yaml @@ -15,7 +15,7 @@ search_performance: - gpu0 - gpu1 wal_enable: true - external_mysql: true + cluster: true readonly: replicas: 3 diff --git a/tests/benchmark/milvus_benchmark/suites/locust_search.yaml b/tests/benchmark/milvus_benchmark/suites/locust_search.yaml index eac80389c3..18fefe32eb 100644 --- a/tests/benchmark/milvus_benchmark/suites/locust_search.yaml +++ b/tests/benchmark/milvus_benchmark/suites/locust_search.yaml @@ -18,7 +18,7 @@ locust_search_performance: - gpu1 wal_enable: true cluster: true - external_mysql: true + readonly: replicas: 1 collection_name: sift_1m_128_l2_2 diff --git a/tests/python_client/testcases/stability/test_mysql.py b/tests/python_client/testcases/stability/test_mysql.py deleted file mode 100644 index 6d308c7697..0000000000 --- a/tests/python_client/testcases/stability/test_mysql.py +++ /dev/null @@ -1,45 +0,0 @@ -import time -import logging -import pytest -from pymilvus import IndexType - -from utils.util_pymilvus import get_milvus, gen_vectors, default_dim -from common.common_type import CaseLabel - - -class TestMysql: - """ - ****************************************************************** - The following cases are used to test mysql failure - ****************************************************************** - """ - - @pytest.fixture(scope="function", autouse=True) - def skip_check(self, connect, args): - if args["service_name"].find("shards") != -1: - reason = "Skip restart cases in shards mode" - logging.getLogger().info(reason) - pytest.skip(reason) - - @pytest.mark.tags(CaseLabel.L2) - def _test_kill_mysql_during_index(self, connect, collection, args): - big_nb = 20000 - index_param = {"nlist": 1024, "m": 16} - index_type = IndexType.IVF_PQ - vectors = gen_vectors(big_nb, default_dim) - status, ids = connect.bulk_insert(collection, vectors, ids=[i for i in range(big_nb)]) - status = connect.flush([collection]) - assert status.OK() - status, res_count = connect.count_entities(collection) - logging.getLogger().info(res_count) - assert status.OK() - assert res_count == big_nb - logging.getLogger().info("Start create index async") - status = connect.create_index(collection, index_type, index_param, _async=True) - time.sleep(2) - logging.getLogger().info("Start play mysql failure") - # pass - new_connect = get_milvus(args["ip"], args["port"], handler=args["handler"]) - status, res_count = new_connect.count_entities(collection) - assert status.OK() - assert res_count == big_nb diff --git a/tests/python_client/testcases/test_alias.py b/tests/python_client/testcases/test_alias.py index 0942f31686..15b3df5674 100644 --- a/tests/python_client/testcases/test_alias.py +++ b/tests/python_client/testcases/test_alias.py @@ -448,7 +448,6 @@ class TestAliasOperationInvalid(TestcaseBase): # check_items=error) @pytest.mark.tags(CaseLabel.L1) - @pytest.mark.skip("not supported on mysql") def test_alias_drop_not_exist_alias(self): """ target: test collection dropping alias which is not exist diff --git a/tests/python_client/utils/util_pymilvus.py b/tests/python_client/utils/util_pymilvus.py index 8a42cf8896..73ec880874 100644 --- a/tests/python_client/utils/util_pymilvus.py +++ b/tests/python_client/utils/util_pymilvus.py @@ -863,7 +863,7 @@ def restart_server(helm_release_name): # body = {"replicas": 0} pods = v1.list_namespaced_pod(namespace) for i in pods.items: - if i.metadata.name.find(helm_release_name) != -1 and i.metadata.name.find("mysql") == -1: + if i.metadata.name.find(helm_release_name) != -1: pod_name = i.metadata.name break # v1.patch_namespaced_config_map(config_map_name, namespace, body, pretty='true') @@ -886,7 +886,7 @@ def restart_server(helm_release_name): log.error(pod_name_tmp) if pod_name_tmp == pod_name: continue - elif pod_name_tmp.find(helm_release_name) == -1 or pod_name_tmp.find("mysql") != -1: + elif pod_name_tmp.find(helm_release_name) == -1: continue else: status_res = v1.read_namespaced_pod_status(pod_name_tmp, namespace, pretty='true') diff --git a/tests/scripts/values/mysql.yaml b/tests/scripts/values/mysql.yaml deleted file mode 100644 index 74fee9c419..0000000000 --- a/tests/scripts/values/mysql.yaml +++ /dev/null @@ -1,274 +0,0 @@ -mysql: - enabled: true - initdbScripts: - meta.sql: |+ - -- create database - CREATE DATABASE if not exists milvus_meta CHARACTER SET utf8mb4; - - /* - create tables script - - Notices: - 1. id, tenant_id, is_deleted, created_at, updated_at are 5 common columns for all collections. - 2. Query index in community version CANNOT includes tenant_id, since tenant_id is not existed and will miss query index. - */ - - -- collections - CREATE TABLE if not exists milvus_meta.collections ( - id BIGINT NOT NULL AUTO_INCREMENT, - tenant_id VARCHAR(128) DEFAULT NULL, - collection_id BIGINT NOT NULL, - collection_name VARCHAR(256), - description VARCHAR(2048) DEFAULT NULL, - auto_id BOOL DEFAULT FALSE, - shards_num INT, - start_position TEXT, - consistency_level INT, - status INT NOT NULL, - properties VARCHAR(512), - ts BIGINT UNSIGNED DEFAULT 0, - is_deleted BOOL DEFAULT FALSE, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP on update current_timestamp, - PRIMARY KEY (id), - UNIQUE KEY uk_tenant_id_collection_id_ts (tenant_id, collection_id, ts) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -- collection aliases - CREATE TABLE if not exists milvus_meta.collection_aliases ( - id BIGINT NOT NULL AUTO_INCREMENT, - tenant_id VARCHAR(128) DEFAULT NULL, - collection_id BIGINT NOT NULL, - collection_alias VARCHAR(128), - ts BIGINT UNSIGNED DEFAULT 0, - is_deleted BOOL DEFAULT FALSE, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP on update current_timestamp, - PRIMARY KEY (id), - UNIQUE KEY uk_tenant_id_collection_alias_ts (tenant_id, collection_alias, ts), - INDEX idx_tenant_id_collection_id_ts (tenant_id, collection_id, ts) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -- channels - CREATE TABLE if not exists milvus_meta.collection_channels ( - id BIGINT NOT NULL AUTO_INCREMENT, - tenant_id VARCHAR(128) DEFAULT NULL, - collection_id BIGINT NOT NULL, - virtual_channel_name VARCHAR(256) NOT NULL, - physical_channel_name VARCHAR(256) NOT NULL, - removed BOOL DEFAULT FALSE, - ts BIGINT UNSIGNED DEFAULT 0, - is_deleted BOOL DEFAULT FALSE, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP on update current_timestamp, - PRIMARY KEY (id), - UNIQUE KEY uk_tenant_id_collection_id_virtual_channel_name_ts (tenant_id, collection_id, virtual_channel_name, ts), - INDEX idx_tenant_id_collection_id_ts (tenant_id, collection_id, ts) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -- fields - CREATE TABLE if not exists milvus_meta.field_schemas ( - id BIGINT NOT NULL AUTO_INCREMENT, - tenant_id VARCHAR(128) DEFAULT NULL, - field_id BIGINT NOT NULL, - field_name VARCHAR(256) NOT NULL, - is_primary_key BOOL NOT NULL, - description VARCHAR(2048) DEFAULT NULL, - data_type INT UNSIGNED NOT NULL, - type_params VARCHAR(2048), - index_params VARCHAR(2048), - auto_id BOOL NOT NULL, - collection_id BIGINT NOT NULL, - ts BIGINT UNSIGNED DEFAULT 0, - is_deleted BOOL DEFAULT FALSE, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP on update current_timestamp, - PRIMARY KEY (id), - UNIQUE KEY uk_tenant_id_collection_id_field_name_ts (tenant_id, collection_id, field_name, ts), - INDEX idx_tenant_id_collection_id_field_id_ts (tenant_id, collection_id, field_id, ts) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -- partitions - CREATE TABLE if not exists milvus_meta.`partitions` ( - id BIGINT NOT NULL AUTO_INCREMENT, - tenant_id VARCHAR(128) DEFAULT NULL, - partition_id BIGINT NOT NULL, - partition_name VARCHAR(256), - partition_created_timestamp bigint unsigned, - collection_id BIGINT NOT NULL, - status INT NOT NULL, - ts BIGINT UNSIGNED DEFAULT 0, - is_deleted BOOL DEFAULT FALSE, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP on update current_timestamp, - PRIMARY KEY (id), - UNIQUE KEY uk_tenant_id_collection_id_partition_name_ts (tenant_id, collection_id, partition_name, ts), - INDEX idx_tenant_id_collection_id_partition_id_ts (tenant_id, collection_id, partition_id, ts) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -- indexes - CREATE TABLE if not exists milvus_meta.`indexes` ( - id BIGINT NOT NULL AUTO_INCREMENT, - tenant_id VARCHAR(128) DEFAULT NULL, - field_id BIGINT NOT NULL, - collection_id BIGINT NOT NULL, - index_id BIGINT NOT NULL, - index_name VARCHAR(256), - index_params VARCHAR(2048), - user_index_params VARCHAR(2048), - is_auto_index BOOL DEFAULT FALSE, - create_time bigint unsigned, - is_deleted BOOL DEFAULT FALSE, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP on update current_timestamp, - PRIMARY KEY (id), - INDEX idx_tenant_id_collection_id_index_id (tenant_id, collection_id, index_id) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -- index file paths - CREATE TABLE if not exists milvus_meta.index_file_paths ( - id BIGINT NOT NULL AUTO_INCREMENT, - tenant_id VARCHAR(128) DEFAULT NULL, - index_build_id BIGINT NOT NULL, - index_file_path VARCHAR(256), - is_deleted BOOL DEFAULT FALSE, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP on update current_timestamp, - PRIMARY KEY (id), - INDEX idx_tenant_id_index_build_id (tenant_id, index_build_id) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -- segments - CREATE TABLE if not exists milvus_meta.segments ( - id BIGINT NOT NULL AUTO_INCREMENT, - tenant_id VARCHAR(128) DEFAULT NULL, - segment_id BIGINT NOT NULL, - collection_id BIGINT NOT NULL, - partition_id BIGINT NOT NULL, - num_rows BIGINT NOT NULL, - max_row_num INT COMMENT 'estimate max rows', - dm_channel VARCHAR(128) NOT NULL, - dml_position TEXT COMMENT 'checkpoint', - start_position TEXT, - compaction_from VARCHAR(4096) COMMENT 'old segment IDs', - created_by_compaction BOOL, - segment_state TINYINT UNSIGNED NOT NULL, - last_expire_time bigint unsigned COMMENT 'segment assignment expiration time', - dropped_at bigint unsigned, - is_deleted BOOL DEFAULT FALSE, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP on update current_timestamp, - PRIMARY KEY (id), - INDEX idx_tenant_id_collection_id_segment_id (tenant_id, collection_id, segment_id) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -- segment indexes - CREATE TABLE if not exists milvus_meta.segment_indexes ( - id BIGINT NOT NULL AUTO_INCREMENT, - tenant_id VARCHAR(128) DEFAULT NULL, - collection_id BIGINT NOT NULL, - partition_id BIGINT NOT NULL, - segment_id BIGINT NOT NULL, - field_id BIGINT NOT NULL, - index_id BIGINT NOT NULL, - index_build_id BIGINT, - enable_index BOOL NOT NULL, - create_time bigint unsigned, - index_file_keys VARCHAR(4096), - index_size BIGINT UNSIGNED, - `version` INT UNSIGNED, - is_deleted BOOL DEFAULT FALSE COMMENT 'as mark_deleted', - recycled BOOL DEFAULT FALSE COMMENT 'binlog files truly deleted', - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP on update current_timestamp, - PRIMARY KEY (id), - UNIQUE KEY uk_tenant_id_segment_id_index_id (tenant_id, segment_id, index_id), - INDEX idx_tenant_id_collection_id_segment_id_index_id (tenant_id, collection_id, segment_id, index_id) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -- binlog files info - CREATE TABLE if not exists milvus_meta.binlogs ( - id BIGINT NOT NULL AUTO_INCREMENT, - tenant_id VARCHAR(128) DEFAULT NULL, - field_id BIGINT NOT NULL, - segment_id BIGINT NOT NULL, - collection_id BIGINT NOT NULL, - log_type SMALLINT UNSIGNED NOT NULL COMMENT 'binlog、stats binlog、delta binlog', - num_entries BIGINT, - timestamp_from BIGINT UNSIGNED, - timestamp_to BIGINT UNSIGNED, - log_path VARCHAR(256) NOT NULL, - log_size BIGINT NOT NULL, - is_deleted BOOL DEFAULT FALSE, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP on update current_timestamp, - PRIMARY KEY (id), - INDEX idx_tenant_id_segment_id_log_type (tenant_id, segment_id, log_type) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -- users - CREATE TABLE if not exists milvus_meta.credential_users ( - id BIGINT NOT NULL AUTO_INCREMENT, - tenant_id VARCHAR(128) DEFAULT NULL, - username VARCHAR(128) NOT NULL, - encrypted_password VARCHAR(256) NOT NULL, - is_super BOOL NOT NULL DEFAULT false, - is_deleted BOOL NOT NULL DEFAULT false, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP on update current_timestamp, - PRIMARY KEY (id), - INDEX idx_tenant_id_username (tenant_id, username) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -- role - CREATE TABLE if not exists milvus_meta.role ( - id BIGINT NOT NULL AUTO_INCREMENT, - tenant_id VARCHAR(128) DEFAULT NULL, - name VARCHAR(128) NOT NULL, - is_deleted BOOL NOT NULL DEFAULT false, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP on update current_timestamp, - INDEX idx_role_tenant_name (tenant_id, name, is_deleted), - PRIMARY KEY (id) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -- user-role - CREATE TABLE if not exists milvus_meta.user_role ( - id BIGINT NOT NULL AUTO_INCREMENT, - tenant_id VARCHAR(128) DEFAULT NULL, - user_id BIGINT NOT NULL, - role_id BIGINT NOT NULL, - is_deleted BOOL NOT NULL DEFAULT false, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP on update current_timestamp, - INDEX idx_role_mapping_tenant_user_role (tenant_id, user_id, role_id, is_deleted), - PRIMARY KEY (id) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -- grant - CREATE TABLE if not exists milvus_meta.grant ( - id BIGINT NOT NULL AUTO_INCREMENT, - tenant_id VARCHAR(128) DEFAULT NULL, - role_id BIGINT NOT NULL, - object VARCHAR(128) NOT NULL, - object_name VARCHAR(128) NOT NULL, - is_deleted BOOL NOT NULL DEFAULT false, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP on update current_timestamp, - INDEX idx_grant_principal_resource_tenant (tenant_id, role_id, object, object_name, is_deleted), - PRIMARY KEY (id) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; - - -- grant-id - CREATE TABLE if not exists milvus_meta.grant_id ( - id BIGINT NOT NULL AUTO_INCREMENT, - tenant_id VARCHAR(128) DEFAULT NULL, - grant_id BIGINT NOT NULL, - grantor_id BIGINT NOT NULL, - privilege VARCHAR(128) NOT NULL, - is_deleted BOOL NOT NULL DEFAULT false, - created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, - updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP on update current_timestamp, - INDEX idx_grant_id_tenant_grantor (tenant_id, grant_id, grantor_id, is_deleted), - PRIMARY KEY (id) - ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4; \ No newline at end of file