From 08ca0a2ca59338ff827e65c0618ca60fdc260df4 Mon Sep 17 00:00:00 2001 From: PowderLi <135960789+PowderLi@users.noreply.github.com> Date: Wed, 24 Jan 2024 11:35:00 +0800 Subject: [PATCH] feat: support etcd authentication (#30226) issue: #28895 add 3 configuration for ETCD config Signed-off-by: PowderLi --- cmd/tools/migration/backend/etcd.go | 5 ++- cmd/tools/migration/migration/runner.go | 5 ++- internal/distributed/datacoord/service.go | 5 ++- internal/distributed/datanode/service.go | 5 ++- internal/distributed/indexnode/service.go | 5 ++- internal/distributed/proxy/service.go | 5 ++- internal/distributed/querycoord/service.go | 5 ++- internal/distributed/querynode/service.go | 5 ++- internal/distributed/rootcoord/service.go | 5 ++- internal/kv/etcd/metakv_factory.go | 5 ++- .../util/dependency/kv/kv_client_handler.go | 5 ++- pkg/util/etcd/etcd_util.go | 38 +++++++++++++++++++ pkg/util/paramtable/service_param.go | 34 +++++++++++++++++ 13 files changed, 116 insertions(+), 11 deletions(-) diff --git a/cmd/tools/migration/backend/etcd.go b/cmd/tools/migration/backend/etcd.go index 0f5e4d28a5..cc1562d9b9 100644 --- a/cmd/tools/migration/backend/etcd.go +++ b/cmd/tools/migration/backend/etcd.go @@ -20,8 +20,11 @@ func (b etcdBasedBackend) CleanWithPrefix(prefix string) error { } func newEtcdBasedBackend(cfg *configs.MilvusConfig) (*etcdBasedBackend, error) { - etcdCli, err := etcd.GetEtcdClient( + etcdCli, err := etcd.CreateEtcdClient( cfg.EtcdCfg.UseEmbedEtcd.GetAsBool(), + cfg.EtcdCfg.EtcdEnableAuth.GetAsBool(), + cfg.EtcdCfg.EtcdAuthUserName.GetValue(), + cfg.EtcdCfg.EtcdAuthPassword.GetValue(), cfg.EtcdCfg.EtcdUseSSL.GetAsBool(), cfg.EtcdCfg.Endpoints.GetAsStrings(), cfg.EtcdCfg.EtcdTLSCert.GetValue(), diff --git a/cmd/tools/migration/migration/runner.go b/cmd/tools/migration/migration/runner.go index e490a2ed12..7d0d791203 100644 --- a/cmd/tools/migration/migration/runner.go +++ b/cmd/tools/migration/migration/runner.go @@ -68,8 +68,11 @@ func (r *Runner) WatchSessions() { } func (r *Runner) initEtcdCli() { - cli, err := etcd.GetEtcdClient( + cli, err := etcd.CreateEtcdClient( r.cfg.EtcdCfg.UseEmbedEtcd.GetAsBool(), + r.cfg.EtcdCfg.EtcdEnableAuth.GetAsBool(), + r.cfg.EtcdCfg.EtcdAuthUserName.GetValue(), + r.cfg.EtcdCfg.EtcdAuthPassword.GetValue(), r.cfg.EtcdCfg.EtcdUseSSL.GetAsBool(), r.cfg.EtcdCfg.Endpoints.GetAsStrings(), r.cfg.EtcdCfg.EtcdTLSCert.GetValue(), diff --git a/internal/distributed/datacoord/service.go b/internal/distributed/datacoord/service.go index f5ff6de90c..f3e2f412eb 100644 --- a/internal/distributed/datacoord/service.go +++ b/internal/distributed/datacoord/service.go @@ -90,8 +90,11 @@ func (s *Server) init() error { params := paramtable.Get() etcdConfig := ¶ms.EtcdCfg - etcdCli, err := etcd.GetEtcdClient( + etcdCli, err := etcd.CreateEtcdClient( etcdConfig.UseEmbedEtcd.GetAsBool(), + etcdConfig.EtcdEnableAuth.GetAsBool(), + etcdConfig.EtcdAuthUserName.GetValue(), + etcdConfig.EtcdAuthPassword.GetValue(), etcdConfig.EtcdUseSSL.GetAsBool(), etcdConfig.Endpoints.GetAsStrings(), etcdConfig.EtcdTLSCert.GetValue(), diff --git a/internal/distributed/datanode/service.go b/internal/distributed/datanode/service.go index 3efcc56400..9aa1ce2535 100644 --- a/internal/distributed/datanode/service.go +++ b/internal/distributed/datanode/service.go @@ -227,8 +227,11 @@ func (s *Server) init() error { log.Warn("DataNode found available port during init", zap.Int("port", Params.Port.GetAsInt())) } - etcdCli, err := etcd.GetEtcdClient( + etcdCli, err := etcd.CreateEtcdClient( etcdConfig.UseEmbedEtcd.GetAsBool(), + etcdConfig.EtcdEnableAuth.GetAsBool(), + etcdConfig.EtcdAuthUserName.GetValue(), + etcdConfig.EtcdAuthPassword.GetValue(), etcdConfig.EtcdUseSSL.GetAsBool(), etcdConfig.Endpoints.GetAsStrings(), etcdConfig.EtcdTLSCert.GetValue(), diff --git a/internal/distributed/indexnode/service.go b/internal/distributed/indexnode/service.go index 774f0f8e1c..ea513bbf12 100644 --- a/internal/distributed/indexnode/service.go +++ b/internal/distributed/indexnode/service.go @@ -168,8 +168,11 @@ func (s *Server) init() error { return err } - etcdCli, err := etcd.GetEtcdClient( + etcdCli, err := etcd.CreateEtcdClient( etcdConfig.UseEmbedEtcd.GetAsBool(), + etcdConfig.EtcdEnableAuth.GetAsBool(), + etcdConfig.EtcdAuthUserName.GetValue(), + etcdConfig.EtcdAuthPassword.GetValue(), etcdConfig.EtcdUseSSL.GetAsBool(), etcdConfig.Endpoints.GetAsStrings(), etcdConfig.EtcdTLSCert.GetValue(), diff --git a/internal/distributed/proxy/service.go b/internal/distributed/proxy/service.go index 04f4ad85d6..a633ccd233 100644 --- a/internal/distributed/proxy/service.go +++ b/internal/distributed/proxy/service.go @@ -443,8 +443,11 @@ func (s *Server) init() error { serviceName := fmt.Sprintf("Proxy ip: %s, port: %d", Params.IP, Params.Port.GetAsInt()) log.Debug("init Proxy's tracer done", zap.String("service name", serviceName)) - etcdCli, err := etcd.GetEtcdClient( + etcdCli, err := etcd.CreateEtcdClient( etcdConfig.UseEmbedEtcd.GetAsBool(), + etcdConfig.EtcdEnableAuth.GetAsBool(), + etcdConfig.EtcdAuthUserName.GetValue(), + etcdConfig.EtcdAuthPassword.GetValue(), etcdConfig.EtcdUseSSL.GetAsBool(), etcdConfig.Endpoints.GetAsStrings(), etcdConfig.EtcdTLSCert.GetValue(), diff --git a/internal/distributed/querycoord/service.go b/internal/distributed/querycoord/service.go index 05f192004b..c2c73df098 100644 --- a/internal/distributed/querycoord/service.go +++ b/internal/distributed/querycoord/service.go @@ -117,8 +117,11 @@ func (s *Server) init() error { etcdConfig := ¶ms.EtcdCfg rpcParams := ¶ms.QueryCoordGrpcServerCfg - etcdCli, err := etcd.GetEtcdClient( + etcdCli, err := etcd.CreateEtcdClient( etcdConfig.UseEmbedEtcd.GetAsBool(), + etcdConfig.EtcdEnableAuth.GetAsBool(), + etcdConfig.EtcdAuthUserName.GetValue(), + etcdConfig.EtcdAuthPassword.GetValue(), etcdConfig.EtcdUseSSL.GetAsBool(), etcdConfig.Endpoints.GetAsStrings(), etcdConfig.EtcdTLSCert.GetValue(), diff --git a/internal/distributed/querynode/service.go b/internal/distributed/querynode/service.go index a94c68d221..c904cc4e12 100644 --- a/internal/distributed/querynode/service.go +++ b/internal/distributed/querynode/service.go @@ -99,8 +99,11 @@ func (s *Server) init() error { log.Debug("QueryNode", zap.Int("port", Params.Port.GetAsInt())) - etcdCli, err := etcd.GetEtcdClient( + etcdCli, err := etcd.CreateEtcdClient( etcdConfig.UseEmbedEtcd.GetAsBool(), + etcdConfig.EtcdEnableAuth.GetAsBool(), + etcdConfig.EtcdAuthUserName.GetValue(), + etcdConfig.EtcdAuthPassword.GetValue(), etcdConfig.EtcdUseSSL.GetAsBool(), etcdConfig.Endpoints.GetAsStrings(), etcdConfig.EtcdTLSCert.GetValue(), diff --git a/internal/distributed/rootcoord/service.go b/internal/distributed/rootcoord/service.go index 7d8bb0af49..f752d18827 100644 --- a/internal/distributed/rootcoord/service.go +++ b/internal/distributed/rootcoord/service.go @@ -175,8 +175,11 @@ func (s *Server) init() error { rpcParams := ¶ms.RootCoordGrpcServerCfg log.Debug("init params done..") - etcdCli, err := etcd.GetEtcdClient( + etcdCli, err := etcd.CreateEtcdClient( etcdConfig.UseEmbedEtcd.GetAsBool(), + etcdConfig.EtcdEnableAuth.GetAsBool(), + etcdConfig.EtcdAuthUserName.GetValue(), + etcdConfig.EtcdAuthPassword.GetValue(), etcdConfig.EtcdUseSSL.GetAsBool(), etcdConfig.Endpoints.GetAsStrings(), etcdConfig.EtcdTLSCert.GetValue(), diff --git a/internal/kv/etcd/metakv_factory.go b/internal/kv/etcd/metakv_factory.go index aa123de1a7..0f2dd64f41 100644 --- a/internal/kv/etcd/metakv_factory.go +++ b/internal/kv/etcd/metakv_factory.go @@ -53,8 +53,11 @@ func NewWatchKVFactory(rootPath string, etcdCfg *paramtable.EtcdConfig) (kv.Watc } return watchKv, err } - client, err := etcd.GetEtcdClient( + client, err := etcd.CreateEtcdClient( etcdCfg.UseEmbedEtcd.GetAsBool(), + etcdCfg.EtcdEnableAuth.GetAsBool(), + etcdCfg.EtcdAuthUserName.GetValue(), + etcdCfg.EtcdAuthPassword.GetValue(), etcdCfg.EtcdUseSSL.GetAsBool(), etcdCfg.Endpoints.GetAsStrings(), etcdCfg.EtcdTLSCert.GetValue(), diff --git a/internal/util/dependency/kv/kv_client_handler.go b/internal/util/dependency/kv/kv_client_handler.go index 40b013849c..4bb8851996 100644 --- a/internal/util/dependency/kv/kv_client_handler.go +++ b/internal/util/dependency/kv/kv_client_handler.go @@ -62,8 +62,11 @@ func getEtcdAndPath() (*clientv3.Client, string) { // Function that calls the Etcd constructor func createEtcdClient() (*clientv3.Client, error) { cfg := ¶mtable.Get().ServiceParam - return etcd.GetEtcdClient( + return etcd.CreateEtcdClient( cfg.EtcdCfg.UseEmbedEtcd.GetAsBool(), + cfg.EtcdCfg.EtcdEnableAuth.GetAsBool(), + cfg.EtcdCfg.EtcdAuthUserName.GetValue(), + cfg.EtcdCfg.EtcdAuthPassword.GetValue(), cfg.EtcdCfg.EtcdUseSSL.GetAsBool(), cfg.EtcdCfg.Endpoints.GetAsStrings(), cfg.EtcdCfg.EtcdTLSCert.GetValue(), diff --git a/pkg/util/etcd/etcd_util.go b/pkg/util/etcd/etcd_util.go index 93a5114881..b1816e0d9e 100644 --- a/pkg/util/etcd/etcd_util.go +++ b/pkg/util/etcd/etcd_util.go @@ -66,8 +66,21 @@ func GetRemoteEtcdClient(endpoints []string) (*clientv3.Client, error) { }) } +func GetRemoteEtcdClientWithAuth(endpoints []string, userName, password string) (*clientv3.Client, error) { + return clientv3.New(clientv3.Config{ + Endpoints: endpoints, + DialTimeout: 5 * time.Second, + Username: userName, + Password: password, + }) +} + func GetRemoteEtcdSSLClient(endpoints []string, certFile string, keyFile string, caCertFile string, minVersion string) (*clientv3.Client, error) { var cfg clientv3.Config + return GetRemoteEtcdSSLClientWithCfg(endpoints, certFile, keyFile, caCertFile, minVersion, cfg) +} + +func GetRemoteEtcdSSLClientWithCfg(endpoints []string, certFile string, keyFile string, caCertFile string, minVersion string, cfg clientv3.Config) (*clientv3.Client, error) { cfg.Endpoints = endpoints cfg.DialTimeout = 5 * time.Second cert, err := tls.LoadX509KeyPair(certFile, keyFile) @@ -108,6 +121,31 @@ func GetRemoteEtcdSSLClient(endpoints []string, certFile string, keyFile string, return clientv3.New(cfg) } +func CreateEtcdClient( + useEmbedEtcd bool, + enableAuth bool, + userName, + password string, + useSSL bool, + endpoints []string, + certFile string, + keyFile string, + caCertFile string, + minVersion string, +) (*clientv3.Client, error) { + if !enableAuth || useEmbedEtcd { + return GetEtcdClient(useEmbedEtcd, useSSL, endpoints, certFile, keyFile, caCertFile, minVersion) + } + log.Info("create etcd client(enable auth)", + zap.Bool("useSSL", useSSL), + zap.Any("endpoints", endpoints), + zap.String("minVersion", minVersion)) + if useSSL { + return GetRemoteEtcdSSLClientWithCfg(endpoints, certFile, keyFile, caCertFile, minVersion, clientv3.Config{Username: userName, Password: password}) + } + return GetRemoteEtcdClientWithAuth(endpoints, userName, password) +} + func min(a, b int) int { if a < b { return a diff --git a/pkg/util/paramtable/service_param.go b/pkg/util/paramtable/service_param.go index 1ed4add8c1..1861f7afaa 100644 --- a/pkg/util/paramtable/service_param.go +++ b/pkg/util/paramtable/service_param.go @@ -107,6 +107,11 @@ type EtcdConfig struct { UseEmbedEtcd ParamItem `refreshable:"false"` ConfigPath ParamItem `refreshable:"false"` DataDir ParamItem `refreshable:"false"` + + // --- ETCD Authentication --- + EtcdEnableAuth ParamItem `refreshable:"false"` + EtcdAuthUserName ParamItem `refreshable:"false"` + EtcdAuthPassword ParamItem `refreshable:"false"` } func (p *EtcdConfig) Init(base *BaseTable) { @@ -267,6 +272,35 @@ We recommend using version 1.2 and above.`, Export: true, } p.RequestTimeout.Init(base.mgr) + + p.EtcdEnableAuth = ParamItem{ + Key: "etcd.auth.enabled", + DefaultValue: "false", + Version: "2.3.7", + Doc: "Whether to enable authentication", + Export: true, + } + p.EtcdEnableAuth.Init(base.mgr) + + if p.UseEmbedEtcd.GetAsBool() && p.EtcdEnableAuth.GetAsBool() { + panic("embedded etcd can not enable auth") + } + + p.EtcdAuthUserName = ParamItem{ + Key: "etcd.auth.userName", + Version: "2.3.7", + Doc: "username for etcd authentication", + Export: true, + } + p.EtcdAuthUserName.Init(base.mgr) + + p.EtcdAuthPassword = ParamItem{ + Key: "etcd.auth.password", + Version: "2.3.7", + Doc: "password for etcd authentication", + Export: true, + } + p.EtcdAuthPassword.Init(base.mgr) } // /////////////////////////////////////////////////////////////////////////////