milvus/internal/util/hookutil/cipher.go

// Licensed to the LF AI & Data foundation under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hookutil

import (
	"bytes"
	"context"
	"encoding/base64"
	"fmt"
	"plugin"
	"strconv"
	"strings"
	"sync"

	"github.com/cockroachdb/errors"
	"github.com/samber/lo"
	"go.uber.org/atomic"
	"go.uber.org/zap"

	"github.com/milvus-io/milvus-proto/go-api/v2/commonpb"
	"github.com/milvus-io/milvus-proto/go-api/v2/hook"
	"github.com/milvus-io/milvus/pkg/v2/log"
	"github.com/milvus-io/milvus/pkg/v2/proto/indexcgopb"
	"github.com/milvus-io/milvus/pkg/v2/streaming/util/message"
	"github.com/milvus-io/milvus/pkg/v2/util/paramtable"
)

var (
	Cipher                 atomic.Value
	initCipherOnce         sync.Once
	cipherReloadMutex      sync.RWMutex
	ErrCipherPluginMissing = errors.New("cipher plugin is missing")
)

// GetCipher returns singleton hook.Cipher instance.
// If Milvus is not built with cipher plugin, it will return nil
// If Milvus is built with cipher plugin, it will return hook.Cipher
func GetCipher() hook.Cipher {
	InitOnceCipher()
	return Cipher.Load().(cipherContainer).cipher
}

func IsClusterEncyptionEnabled() bool {
	return GetCipher() != nil
}

const (
	// Used in db and collection properties
	EncryptionEnabledKey = "cipher.enabled"
	EncryptionRootKeyKey = "cipher.key"
	EncryptionEzIDKey    = "cipher.ezID"

	// Used in Plugins
	CipherConfigCreateEZ       = "cipher.ez.create"
	CipherConfigRemoveEZ       = "cipher.ez.remove"
	CipherConfigMilvusRoleName = "cipher.milvusRoleName"
	CipherConfigKeyKmsKeyArn   = "cipher.kmsKeyArn"
	CipherConfigUnsafeEZK      = "cipher.ezk"
)

type EZ struct {
	EzID         int64
	CollectionID int64
}

func (ez *EZ) AsMessageConfig() *message.CipherConfig {
	if ez == nil {
		return nil
	}
	return &message.CipherConfig{EzID: ez.EzID, CollectionID: ez.CollectionID}
}

type CipherContext struct {
	EZ
	key []byte
}

func ContainsCipherProperties(properties []*commonpb.KeyValuePair, deletedKeys []string) bool {
	for _, property := range properties {
		if property.Key == EncryptionEnabledKey ||
			property.Key == EncryptionEzIDKey ||
			property.Key == EncryptionRootKeyKey {
			return true
		}
	}
	return lo.ContainsBy(deletedKeys, func(data string) bool {
		return lo.Contains([]string{EncryptionEnabledKey, EncryptionEzIDKey, EncryptionRootKeyKey}, data)
	})
}

func GetEzByCollProperties(collProperties []*commonpb.KeyValuePair, collectionID int64) *EZ {
	if len(collProperties) == 0 {
		return nil
	}
	for _, property := range collProperties {
		if property.Key == EncryptionEzIDKey {
			ezID, _ := strconv.ParseInt(property.Value, 10, 64)
			return &EZ{
				EzID:         ezID,
				CollectionID: collectionID,
			}
		}
	}
	return nil
}

// GetStoragePluginContext returns the local plugin context for RPC from datacoord to datanode
func GetStoragePluginContext(properties []*commonpb.KeyValuePair, collectionID int64) []*commonpb.KeyValuePair {
	if GetCipher() == nil {
		return nil
	}

	if ez := GetEzByCollProperties(properties, collectionID); ez != nil {
		key := GetCipher().GetUnsafeKey(ez.EzID, ez.CollectionID)
		pluginContext := []*commonpb.KeyValuePair{
			{
				Key:   CipherConfigCreateEZ,
				Value: strconv.FormatInt(ez.EzID, 10),
			},
			{
				Key:   CipherConfigUnsafeEZK,
				Value: base64.StdEncoding.EncodeToString(key),
			},
		}
		return pluginContext
	}

	return nil
}

func GetDBCipherProperties(ezID uint64, kmsKey string) []*commonpb.KeyValuePair {
	return []*commonpb.KeyValuePair{
		{
			Key:   EncryptionEnabledKey,
			Value: "true",
		},
		{
			Key:   EncryptionEzIDKey,
			Value: strconv.FormatUint(ezID, 10),
		},
		{
			Key:   EncryptionRootKeyKey,
			Value: kmsKey,
		},
	}
}

func RemoveEZByDBProperties(dbProperties []*commonpb.KeyValuePair) error {
	if GetCipher() == nil {
		return nil
	}

	ezIdStr := ""
	for _, property := range dbProperties {
		if property.Key == EncryptionEzIDKey {
			ezIdStr = property.Value
		}
	}
	if len(ezIdStr) == 0 {
		return nil
	}

	dropConfig := map[string]string{CipherConfigRemoveEZ: ezIdStr}
	if err := GetCipher().Init(dropConfig); err != nil {
		return err
	}
	return nil
}

func CreateLocalEZByPluginContext(context []*commonpb.KeyValuePair) (*indexcgopb.StoragePluginContext, error) {
	if GetCipher() == nil {
		return nil, nil
	}
	config := make(map[string]string)
	ctx := &indexcgopb.StoragePluginContext{}
	for _, value := range context {
		if value.GetKey() == CipherConfigCreateEZ {
			ezID, err := strconv.ParseInt(value.GetValue(), 10, 64)
			if err != nil {
				return nil, err
			}
			config[CipherConfigCreateEZ] = value.GetValue()
			ctx.EncryptionZoneId = ezID
		}
		if value.GetKey() == CipherConfigUnsafeEZK {
			config[CipherConfigUnsafeEZK] = value.GetValue()
			ctx.EncryptionKey = value.GetValue()
		}
	}
	if len(config) == 2 {
		return ctx, GetCipher().Init(config)
	}
	return nil, nil
}

func CreateEZByDBProperties(dbProperties []*commonpb.KeyValuePair) error {
	if GetCipher() == nil {
		return nil
	}

	config := make(map[string]string)
	for _, property := range dbProperties {
		if property.GetKey() == EncryptionEzIDKey {
			config[CipherConfigCreateEZ] = property.Value
		}
		if property.GetKey() == EncryptionRootKeyKey {
			config[CipherConfigKeyKmsKeyArn] = property.GetValue()
		}
	}

	if len(config) == 2 {
		return GetCipher().Init(config)
	}

	return nil
}

func TidyDBCipherProperties(ezID int64, dbProperties []*commonpb.KeyValuePair) ([]*commonpb.KeyValuePair, error) {
	dbEncryptionEnabled := IsDBEncryptionEnabled(dbProperties)
	if GetCipher() == nil {
		if dbEncryptionEnabled {
			return nil, ErrCipherPluginMissing
		}
		return dbProperties, nil
	}

	if dbEncryptionEnabled {
		ezIDKv := &commonpb.KeyValuePair{
			Key:   EncryptionEzIDKey,
			Value: strconv.FormatInt(ezID, 10),
		}
		// kmsKey already in the properties
		for _, property := range dbProperties {
			if property.Key == EncryptionRootKeyKey {
				dbProperties = append(dbProperties, ezIDKv)
				return dbProperties, nil
			}
		}

		if defaultRootKey := paramtable.GetCipherParams().DefaultRootKey.GetValue(); defaultRootKey != "" {
			// set default root key from config if EncryuptionRootKeyKey left empty
			dbProperties = append(dbProperties,
				ezIDKv,
				&commonpb.KeyValuePair{
					Key:   EncryptionRootKeyKey,
					Value: defaultRootKey,
				},
			)
			return dbProperties, nil
		}
		return nil, fmt.Errorf("Empty default root key for encrypted database without kms key")
	}
	return dbProperties, nil
}

func GetEzPropByDBProperties(dbProperties []*commonpb.KeyValuePair) *commonpb.KeyValuePair {
	for _, property := range dbProperties {
		if property.Key == EncryptionEzIDKey {
			return &commonpb.KeyValuePair{
				Key:   EncryptionEzIDKey,
				Value: property.Value,
			}
		}
	}
	return nil
}

func IsDBEncryptionEnabled(dbProperties []*commonpb.KeyValuePair) bool {
	for _, property := range dbProperties {
		if property.Key == EncryptionEnabledKey && strings.ToLower(property.Value) == "true" {
			return true
		}
	}
	return false
}

// For test only
func InitTestCipher() {
	InitOnceCipher()
	storeCipher(testCipher{})
}

// cipherContainer is Container to wrap hook.Cipher interface
// this struct is used to be stored in atomic.Value
// since different type stored in it will cause panicking.
type cipherContainer struct {
	cipher hook.Cipher
}

func storeCipher(cipher hook.Cipher) {
	Cipher.Store(cipherContainer{cipher: cipher})
}

func initCipher() error {
	storeCipher(nil)

	pathGo := paramtable.GetCipherParams().SoPathGo.GetValue()
	pathCpp := paramtable.GetCipherParams().SoPathCpp.GetValue()
	if pathGo == "" || pathCpp == "" {
		log.Info("empty so path for cipher plugin, skip to load plugin")
		return nil
	}

	log.Info("start to load cipher go plugin", zap.String("path", pathGo))
	p, err := plugin.Open(pathGo)
	if err != nil {
		return fmt.Errorf("fail to open the cipher plugin, error: %s", err.Error())
	}
	log.Info("cipher plugin opened", zap.String("path", pathGo))

	h, err := p.Lookup("CipherPlugin")
	if err != nil {
		return fmt.Errorf("fail to the 'CipherPlugin' object in the plugin, error: %s", err.Error())
	}

	cipherVal, ok := h.(hook.Cipher)
	if !ok {
		return fmt.Errorf("fail to convert the `CipherPlugin` interface")
	}

	initConfigs := buildCipherInitConfig()
	if err = cipherVal.Init(initConfigs); err != nil {
		return fmt.Errorf("fail to init configs for the cipher plugin, error: %s", err.Error())
	}

	registerCallback()
	storeCipher((cipherVal))
	return nil
}

func InitOnceCipher() {
	initCipherOnce.Do(func() {
		err := initCipher()
		if err != nil {
			log.Panic("fail to init cipher plugin",
				zap.String("Go so path", paramtable.GetCipherParams().SoPathGo.GetValue()),
				zap.String("Cpp so path", paramtable.GetCipherParams().SoPathCpp.GetValue()),
				zap.Error(err))
		}
	})
}

func buildCipherInitConfig() map[string]string {
	initConfigs := lo.Assign(paramtable.Get().EtcdCfg.GetAll(), paramtable.GetCipherParams().GetAll())
	initConfigs[CipherConfigMilvusRoleName] = paramtable.GetRole()
	initConfigs[paramtable.Get().ServiceParam.LocalStorageCfg.Path.Key] = paramtable.Get().ServiceParam.LocalStorageCfg.Path.GetValue()
	return initConfigs
}

func registerCallback() {
	params := paramtable.GetCipherParams()
	params.DefaultRootKey.RegisterCallback(reloadCipherConfig)
	params.KmsAwsRoleARN.RegisterCallback(reloadCipherConfig)
	params.KmsAwsExternalID.RegisterCallback(reloadCipherConfig)
	params.RotationPeriodInHours.RegisterCallback(reloadCipherConfig)
	params.UpdatePerieldInMinutes.RegisterCallback(reloadCipherConfig)
	log.Info("cipher config callbacks registered")
}

func reloadCipherConfig(ctx context.Context, key, oldValue, newValue string) error {
	cipher := GetCipher()
	if cipher == nil {
		log.Warn("cipher plugin not loaded, skip config reload", zap.String("key", key))
		return nil
	}

	cipherReloadMutex.Lock()
	defer cipherReloadMutex.Unlock()

	log.Info("reloading cipher plugin config",
		zap.String("key", key),
		zap.String("oldValue", oldValue),
		zap.String("newValue", newValue))

	initConfigs := buildCipherInitConfig()
	if err := cipher.Init(initConfigs); err != nil {
		log.Error("fail to reload cipher plugin config",
			zap.String("key", key),
			zap.Error(err))
		return err
	}

	log.Info("cipher plugin config reloaded successfully", zap.String("key", key))
	return nil
}

// testCipher encryption will append magicStr to plainText, magicStr is str of ezID and collectionID
type testCipher struct{}

var (
	_ hook.Cipher    = (*testCipher)(nil)
	_ hook.Encryptor = (*testCryptoImpl)(nil)
	_ hook.Decryptor = (*testCryptoImpl)(nil)
)

func (d testCipher) Init(params map[string]string) error {
	return nil
}

func (d testCipher) GetEncryptor(ezID, collectionID int64) (encryptor hook.Encryptor, safeKey []byte, err error) {
	return createTestCryptoImpl(ezID, collectionID), []byte("safe key"), nil
}

func (d testCipher) GetDecryptor(ezID, collectionID int64, safeKey []byte) (hook.Decryptor, error) {
	return createTestCryptoImpl(ezID, collectionID), nil
}

func (d testCipher) GetUnsafeKey(ezID, collectionID int64) []byte {
	return []byte("unsafe key")
}

// append magicStr to plainText
type testCryptoImpl struct {
	magicStr string
}

func createTestCryptoImpl(ezID, collectionID int64) testCryptoImpl {
	return testCryptoImpl{fmt.Sprintf("%d%d", ezID, collectionID)}
}

func (c testCryptoImpl) Encrypt(plainText []byte) (cipherText []byte, err error) {
	return append(plainText, []byte(c.magicStr)...), nil
}

func (c testCryptoImpl) Decrypt(cipherText []byte) (plainText []byte, err error) {
	return bytes.TrimSuffix(cipherText, []byte(c.magicStr)), nil
}