diff --git a/configs/milvus.yaml b/configs/milvus.yaml index ccc93df580..8f3dfdf6c7 100644 --- a/configs/milvus.yaml +++ b/configs/milvus.yaml @@ -297,6 +297,9 @@ common: # please adjust in embedded Milvus: local storageType: minio + security: authorizationEnabled: false - tlsEnabled: false + # tls mode values [0, 1, 2] + # 0 is close, 1 is one-way authentication, 2 is two-way authentication. + tlsMode: 0 diff --git a/docs/user_guides/tls_proxy.md b/docs/user_guides/tls_proxy.md index dbaf0b379a..aad7689952 100644 --- a/docs/user_guides/tls_proxy.md +++ b/docs/user_guides/tls_proxy.md @@ -1,6 +1,6 @@ # How to enable use TLS proxy -Milvus proxy uses TLS mutual authentication. +Milvus proxy uses TLS two-way and one-way authentication. @@ -489,8 +489,14 @@ tls: common: security: - tlsEnabled: true + tlsMode: 2 ``` +### One-way authentication +Server need server.pem and server.key. Client-side need server.pem. +### Two-way authentication +Server-side need server.pem, server.key and ca.pem. Client-side need client.pem, client.key, ca.pem. + + ## Connect to the Milvus server with TLS diff --git a/internal/distributed/proxy/service.go b/internal/distributed/proxy/service.go index 2a133dda31..ec4809807f 100644 --- a/internal/distributed/proxy/service.go +++ b/internal/distributed/proxy/service.go @@ -188,22 +188,33 @@ func (s *Server) startExternalGrpc(grpcPort int, errChan chan error) { grpc_auth.StreamServerInterceptor(proxy.AuthenticationInterceptor))), } - if Params.TLSEnabled { + if Params.TLSMode == 1 { + creds, err := credentials.NewServerTLSFromFile(Params.ServerPemPath, Params.ServerKeyPath) + if err != nil { + log.Warn("proxy can't create creds", zap.Error(err)) + errChan <- err + return + } + grpcOpts = append(grpcOpts, grpc.Creds(creds)) + } else if Params.TLSMode == 2 { cert, err := tls.LoadX509KeyPair(Params.ServerPemPath, Params.ServerKeyPath) if err != nil { log.Warn("proxy cant load x509 key pair", zap.Error(err)) - panic(err) + errChan <- err + return } certPool := x509.NewCertPool() rootBuf, err := ioutil.ReadFile(Params.CaPemPath) if err != nil { log.Warn("failed read ca pem", zap.Error(err)) - panic(err) + errChan <- err + return } if !certPool.AppendCertsFromPEM(rootBuf) { log.Warn("fail to append ca to cert") - panic("fail to append ca to cert") + errChan <- fmt.Errorf("fail to append ca to cert") + return } tlsConf := &tls.Config{ @@ -226,7 +237,8 @@ func (s *Server) startExternalGrpc(grpcPort int, errChan chan error) { if err := s.grpcExternalServer.Serve(lis); err != nil { log.Error("failed to serve on Proxy's listener", zap.Error(err)) - panic(err) + errChan <- err + return } } @@ -277,7 +289,8 @@ func (s *Server) startInternalGrpc(grpcPort int, errChan chan error) { if err := s.grpcInternalServer.Serve(lis); err != nil { log.Error("failed to internal serve on Proxy's listener", zap.Error(err)) - panic(err) + errChan <- err + return } } diff --git a/internal/distributed/proxy/service_test.go b/internal/distributed/proxy/service_test.go index f9ae640dc1..2dc9205b54 100644 --- a/internal/distributed/proxy/service_test.go +++ b/internal/distributed/proxy/service_test.go @@ -805,7 +805,7 @@ func (m *MockProxy) ListCredUsers(ctx context.Context, req *milvuspb.ListCredUse type WaitOption struct { Duration time.Duration `json:"duration"` Port int `json:"port"` - TLSEnabled bool `json:"tls_enabled"` + TLSMode int `json:"tls_mode"` ClientPemPath string `json:"client_pem_path"` ClientKeyPath string `json:"client_key_path"` CaPath string `json:"ca_path"` @@ -819,11 +819,11 @@ func (opt *WaitOption) String() string { return string(s) } -func newWaitOption(duration time.Duration, Port int, tlsEnabled bool, clientPemPath, clientKeyPath, clientCaPath string) *WaitOption { +func newWaitOption(duration time.Duration, Port int, tlsMode int, clientPemPath, clientKeyPath, clientCaPath string) *WaitOption { return &WaitOption{ Duration: duration, Port: Port, - TLSEnabled: tlsEnabled, + TLSMode: tlsMode, ClientPemPath: clientPemPath, ClientKeyPath: clientKeyPath, CaPath: clientCaPath, @@ -860,16 +860,25 @@ func waitForGrpcReady(opt *WaitOption) { go func() { // just used in UT to self-check service is available. address := "localhost:" + strconv.Itoa(opt.Port) - if !opt.TLSEnabled { - _, err := grpc.Dial(address, grpc.WithBlock(), grpc.WithInsecure()) - ch <- err - } else { - creds, err := withCredential(opt.ClientPemPath, opt.ClientKeyPath, opt.CaPath) + var err error + + if opt.TLSMode == 1 || opt.TLSMode == 2 { + var creds credentials.TransportCredentials + if opt.TLSMode == 1 { + creds, err = credentials.NewClientTLSFromFile(Params.ServerPemPath, "localhost") + } else { + creds, err = withCredential(opt.ClientPemPath, opt.ClientKeyPath, opt.CaPath) + } if err != nil { ch <- err + return } _, err = grpc.Dial(address, grpc.WithBlock(), grpc.WithTransportCredentials(creds)) ch <- err + return + } + if _, err := grpc.Dial(address, grpc.WithBlock(), grpc.WithInsecure()); true { + ch <- err } }() @@ -895,8 +904,8 @@ var clientKeyPath = "../../../configs/cert/client.key" // waitForServerReady wait for internal grpc service and external service to be ready, according to the params. func waitForServerReady() { - waitForGrpcReady(newWaitOption(waitDuration, Params.InternalPort, false, "", "", "")) - waitForGrpcReady(newWaitOption(waitDuration, Params.Port, Params.TLSEnabled, clientPemPath, clientKeyPath, Params.CaPemPath)) + waitForGrpcReady(newWaitOption(waitDuration, Params.InternalPort, 0, "", "", "")) + waitForGrpcReady(newWaitOption(waitDuration, Params.Port, Params.TLSMode, clientPemPath, clientKeyPath, Params.CaPemPath)) } func runAndWaitForServerReady(server *Server) error { @@ -1184,7 +1193,6 @@ func Test_NewServer(t *testing.T) { proxy.Params.ProxyCfg.GinLogging = false err = runAndWaitForServerReady(server) assert.Nil(t, err) - err = server.Stop() assert.Nil(t, err) } @@ -1329,11 +1337,11 @@ func Test_NewServer_HTTPServerDisabled(t *testing.T) { err = runAndWaitForServerReady(server) assert.Nil(t, err) assert.Nil(t, server.httpServer) - err = server.Stop() assert.Nil(t, err) } -func Test_NewServer_TLS(t *testing.T) { + +func getServer(t *testing.T) *Server { ctx := context.Background() server, err := NewServer(ctx, nil) assert.NotNil(t, server) @@ -1344,15 +1352,67 @@ func Test_NewServer_TLS(t *testing.T) { server.indexCoordClient = &MockIndexCoord{} server.queryCoordClient = &MockQueryCoord{} server.dataCoordClient = &MockDataCoord{} + return server +} - Params.TLSEnabled = true +func Test_NewServer_TLS_TwoWay(t *testing.T) { + server := getServer(t) + + Params.InitOnce("proxy") + Params.TLSMode = 2 Params.ServerPemPath = "../../../configs/cert/server.pem" Params.ServerKeyPath = "../../../configs/cert/server.key" Params.CaPemPath = "../../../configs/cert/ca.pem" - err = runAndWaitForServerReady(server) + err := runAndWaitForServerReady(server) assert.Nil(t, err) - + assert.NotNil(t, server.grpcExternalServer) err = server.Stop() assert.Nil(t, err) } + +func Test_NewServer_TLS_OneWay(t *testing.T) { + server := getServer(t) + + Params.InitOnce("proxy") + Params.TLSMode = 1 + Params.ServerPemPath = "../../../configs/cert/server.pem" + Params.ServerKeyPath = "../../../configs/cert/server.key" + + err := runAndWaitForServerReady(server) + assert.Nil(t, err) + assert.NotNil(t, server.grpcExternalServer) + err = server.Stop() + assert.Nil(t, err) +} + +func Test_NewServer_TLS_FileNotExisted(t *testing.T) { + server := getServer(t) + + Params.InitOnce("proxy") + Params.TLSMode = 1 + Params.ServerPemPath = "../not/existed/server.pem" + Params.ServerKeyPath = "../../../configs/cert/server.key" + err := runAndWaitForServerReady(server) + assert.NotNil(t, err) + server.Stop() + + Params.TLSMode = 2 + Params.ServerPemPath = "../not/existed/server.pem" + Params.CaPemPath = "../../../configs/cert/ca.pem" + err = runAndWaitForServerReady(server) + assert.NotNil(t, err) + server.Stop() + + Params.ServerPemPath = "../../../configs/cert/server.pem" + Params.CaPemPath = "../not/existed/ca.pem" + err = runAndWaitForServerReady(server) + assert.NotNil(t, err) + server.Stop() + + Params.ServerPemPath = "../../../configs/cert/server.pem" + Params.CaPemPath = "service.go" + err = runAndWaitForServerReady(server) + assert.NotNil(t, err) + server.Stop() +} diff --git a/internal/util/paramtable/grpc_param.go b/internal/util/paramtable/grpc_param.go index a10596d6d4..d67dde0929 100644 --- a/internal/util/paramtable/grpc_param.go +++ b/internal/util/paramtable/grpc_param.go @@ -55,7 +55,7 @@ type grpcConfig struct { once sync.Once Domain string IP string - TLSEnabled bool + TLSMode int Port int InternalPort int ServerPemPath string @@ -89,7 +89,7 @@ func (p *grpcConfig) initPort() { } func (p *grpcConfig) initTLSPath() { - p.TLSEnabled = p.ParseBool("common.security.tlsEnabled", false) + p.TLSMode = p.ParseIntWithDefault("common.security.tlsMode", 0) p.ServerPemPath = p.Get("tls.serverPemPath") p.ServerKeyPath = p.Get("tls.serverKeyPath") p.CaPemPath = p.Get("tls.caPemPath") diff --git a/internal/util/paramtable/grpc_param_test.go b/internal/util/paramtable/grpc_param_test.go index 26aaf6b87a..668503fa97 100644 --- a/internal/util/paramtable/grpc_param_test.go +++ b/internal/util/paramtable/grpc_param_test.go @@ -117,4 +117,13 @@ func TestGrpcClientParams(t *testing.T) { Params.initKeepAliveTimeout() assert.Equal(t, Params.KeepAliveTimeout, 500*time.Millisecond) + Params.Save("common.security.tlsMode", "1") + Params.Save("tls.serverPemPath", "/pem") + Params.Save("tls.serverKeyPath", "/key") + Params.Save("tls.caPemPath", "/ca") + Params.initTLSPath() + assert.Equal(t, Params.TLSMode, 1) + assert.Equal(t, Params.ServerPemPath, "/pem") + assert.Equal(t, Params.ServerKeyPath, "/key") + assert.Equal(t, Params.CaPemPath, "/ca") }