Cloudpods/backend/pkg/keystone/models/configs.go

// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package models

import (
	"context"
	"database/sql"
	"sort"
	"time"

	"yunion.io/x/jsonutils"
	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"
	"yunion.io/x/pkg/gotypes"
	"yunion.io/x/pkg/utils"

	api "yunion.io/x/onecloud/pkg/apis/identity"
	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
	"yunion.io/x/onecloud/pkg/cloudcommon/db"
	common_options "yunion.io/x/onecloud/pkg/cloudcommon/options"
	"yunion.io/x/onecloud/pkg/httperrors"
	"yunion.io/x/onecloud/pkg/mcclient"
	"yunion.io/x/onecloud/pkg/util/logclient"
)

// +onecloud:swagger-gen-ignore
type SConfigOptionManager struct {
	db.SResourceBaseManager
	db.SRecordChecksumResourceBaseManager
	IsSensitive bool
}

var (
	SensitiveConfigManager   *SConfigOptionManager
	WhitelistedConfigManager *SConfigOptionManager
)

func init() {
	SensitiveConfigManager = &SConfigOptionManager{
		SResourceBaseManager: db.NewResourceBaseManager(
			SConfigOption{},
			"sensitive_config",
			"sensitive_config",
			"sensitive_configs",
		),
		SRecordChecksumResourceBaseManager: *db.NewRecordChecksumResourceBaseManager(),
		IsSensitive:                        true,
	}
	SensitiveConfigManager.SetVirtualObject(SensitiveConfigManager)

	WhitelistedConfigManager = &SConfigOptionManager{
		SResourceBaseManager: db.NewResourceBaseManager(
			SConfigOption{},
			"whitelisted_config",
			"whitelisted_config",
			"whitelisted_configs",
		),
		SRecordChecksumResourceBaseManager: *db.NewRecordChecksumResourceBaseManager(),
		IsSensitive:                        false,
	}
	WhitelistedConfigManager.SetVirtualObject(WhitelistedConfigManager)
}

/*
+-----------+--------------+------+-----+---------+-------+
| Field     | Type         | Null | Key | Default | Extra |
+-----------+--------------+------+-----+---------+-------+
| domain_id | varchar(64)  | NO   | PRI | NULL    |       |
| group     | varchar(255) | NO   | PRI | NULL    |       |
| option    | varchar(255) | NO   | PRI | NULL    |       |
| value     | text         | NO   |     | NULL    |       |
+-----------+--------------+------+-----+---------+-------+
*/

type SConfigOption struct {
	db.SResourceBase
	db.SRecordChecksumResourceBase

	ResType string `width:"32" charset:"ascii" nullable:"false" default:"identity_provider" primary:"true"`
	ResId   string `name:"domain_id" width:"64" charset:"ascii" primary:"true"`
	Group   string `width:"191" charset:"utf8" primary:"true"`
	Option  string `width:"191" charset:"utf8" primary:"true"`

	Value jsonutils.JSONObject `nullable:"false"`
}

func (manager *SConfigOptionManager) fetchConfigs(model db.IModel, groups []string, options []string) (TConfigOptions, error) {
	return manager.fetchConfigs2(model.Keyword(), model.GetId(), groups, options)
}

func (manager *SConfigOptionManager) fetchConfigs2(resType string, resId string, groups []string, options []string) (TConfigOptions, error) {
	q := manager.Query()
	if len(resType) > 0 {
		q = q.Equals("res_type", resType)
	}
	if len(resId) > 0 {
		q = q.Equals("domain_id", resId)
	}
	if len(groups) > 0 {
		q = q.In("group", groups)
	}
	if len(options) > 0 {
		q = q.In("option", options)
	}
	opts := make(TConfigOptions, 0)
	err := db.FetchModelObjects(manager, q, &opts)
	if err != nil && err != sql.ErrNoRows {
		return nil, errors.Wrap(err, "FetchModelObjects")
	}
	sort.Sort(opts)
	return opts, nil
}

func config2map(opts []SConfigOption) api.TConfigs {
	conf := make(api.TConfigs)
	for i := range opts {
		opt := opts[i]
		if _, ok := conf[opt.Group]; !ok {
			conf[opt.Group] = make(map[string]jsonutils.JSONObject)
		}
		conf[opt.Group][opt.Option] = opt.Value
	}
	return conf
}

func (manager *SConfigOptionManager) deleteConfigs(model db.IModel) ([]sChangeOption, error) {
	return manager.syncConfigs(model, nil)
}

type sChangeOption struct {
	Group  string               `json:"group"`
	Option string               `json:"option"`
	Value  jsonutils.JSONObject `json:"value"`
	OValue jsonutils.JSONObject `json:"ovalue"`
}

func (manager *SConfigOptionManager) updateConfigs(model db.IModel, newOpts TConfigOptions) ([]sChangeOption, error) {
	return manager.syncRemoveConfigs(model, newOpts, false)
}

func (manager *SConfigOptionManager) removeConfigs(model db.IModel, newOpts TConfigOptions) ([]sChangeOption, error) {
	oldOpts, err := manager.fetchConfigs(model, nil, nil)
	if err != nil {
		return nil, errors.Wrap(err, "fetchOldConfigs")
	}
	changed := make([]sChangeOption, 0)
	for i := range newOpts {
		newOpts[i].Value = nil
	}
	_, updated1, _, _ := compareConfigOptions(oldOpts, newOpts)
	for i := range updated1 {
		_, err := db.Update(&updated1[i], func() error {
			return updated1[i].MarkDelete()
		})
		if err != nil {
			return changed, errors.Wrap(err, "Delete")
		}
		changed = append(changed, sChangeOption{
			Group:  updated1[i].Group,
			Option: updated1[i].Option,
			OValue: updated1[i].Value,
		})
	}
	return changed, nil
}

func (manager *SConfigOptionManager) syncConfigs(model db.IModel, newOpts TConfigOptions) ([]sChangeOption, error) {
	return manager.syncRemoveConfigs(model, newOpts, true)
}

func (manager *SConfigOptionManager) syncRemoveConfigs(model db.IModel, newOpts TConfigOptions, remove bool) ([]sChangeOption, error) {
	oldOpts, err := manager.fetchConfigs(model, nil, nil)
	if err != nil {
		return nil, errors.Wrap(err, "fetchOldConfigs")
	}
	changed := make([]sChangeOption, 0)
	deleted, updated1, updated2, added := compareConfigOptions(oldOpts, newOpts)
	if remove {
		for i := range deleted {
			_, err := db.Update(&deleted[i], func() error {
				return deleted[i].MarkDelete()
			})
			if err != nil {
				return changed, errors.Wrap(err, "Delete")
			}
			changed = append(changed, sChangeOption{
				Group:  deleted[i].Group,
				Option: deleted[i].Option,
				OValue: deleted[i].Value,
			})
		}
	}
	for i := range updated1 {
		ovalue := updated1[i].Value
		_, err = db.Update(&updated1[i], func() error {
			updated1[i].Value = updated2[i].Value
			return nil
		})
		if err != nil {
			return changed, errors.Wrap(err, "Update")
		}
		changed = append(changed, sChangeOption{
			Group:  updated1[i].Group,
			Option: updated1[i].Option,
			OValue: ovalue,
			Value:  updated2[i].Value,
		})
	}
	for i := range added {
		err = manager.TableSpec().InsertOrUpdate(context.TODO(), &added[i])
		if err != nil {
			return changed, errors.Wrap(err, "Insert")
		}
		changed = append(changed, sChangeOption{
			Group:  added[i].Group,
			Option: added[i].Option,
			Value:  added[i].Value,
		})
	}
	return changed, nil
}

func filterOptions(opts TConfigOptions, whiteList map[string][]string, blackList map[string][]string) TConfigOptions {
	if whiteList == nil && blackList == nil {
		return opts
	}
	retOpts := make(TConfigOptions, 0)
	for _, opt := range opts {
		if whiteList != nil {
			if v, ok := whiteList[opt.Group]; ok && utils.IsInStringArray(opt.Option, v) {
				retOpts = append(retOpts, opt)
			}
		} else if blackList != nil {
			if v, ok := blackList[opt.Group]; ok && utils.IsInStringArray(opt.Option, v) {
			} else {
				retOpts = append(retOpts, opt)
			}
		}
	}
	return retOpts
}

func getConfigOptions(conf api.TConfigs, model db.IModel, sensitiveList map[string][]string) (TConfigOptions, TConfigOptions) {
	options := make(TConfigOptions, 0)
	sensitive := make(TConfigOptions, 0)
	for group, groupConf := range conf {
		for optKey, optVal := range groupConf {
			opt := &SConfigOption{
				ResType: model.Keyword(),
				ResId:   model.GetId(),
				Group:   group,
				Option:  optKey,
				Value:   optVal,
			}
			opt.SetModelManager(WhitelistedConfigManager, opt)
			if v, ok := sensitiveList[group]; ok && utils.IsInStringArray(optKey, v) {
				sensitive = append(sensitive, *opt)
			} else {
				options = append(options, *opt)
			}
		}
	}
	sort.Sort(options)
	sort.Sort(sensitive)
	return options, sensitive
}

type TConfigOptions []SConfigOption

func (opts TConfigOptions) Validate() error {
	for _, opt := range opts {
		if opt.Option == "time_zone" && opt.Group == "default" && !gotypes.IsNil(opt.Value) {
			timeZone, _ := opt.Value.GetString()
			_, err := time.LoadLocation(timeZone)
			if err != nil {
				return httperrors.NewInputParameterError("invalid time_zone %s", timeZone)
			}
		}
	}
	return nil
}

func (opts TConfigOptions) Len() int {
	return len(opts)
}

func (opts TConfigOptions) Swap(i, j int) {
	opts[i], opts[j] = opts[j], opts[i]
}

func (opts TConfigOptions) Less(i, j int) bool {
	return compareConfigOption(opts[i], opts[j]) < 0
}

func compareConfigOption(opt1, opt2 SConfigOption) int {
	if opt1.Group < opt2.Group {
		return -1
	} else if opt1.Group > opt2.Group {
		return 1
	}
	if opt1.Option < opt2.Option {
		return -1
	} else if opt1.Option > opt2.Option {
		return 1
	}
	return 0
}

func compareConfigOptions(opts1, opts2 TConfigOptions) (deleted, updated1, updated2, added TConfigOptions) {
	deleted = make(TConfigOptions, 0)
	updated1 = make(TConfigOptions, 0)
	updated2 = make(TConfigOptions, 0)
	added = make(TConfigOptions, 0)

	i := 0
	j := 0

	for i < len(opts1) && j < len(opts2) {
		cmp := compareConfigOption(opts1[i], opts2[j])
		if cmp < 0 {
			deleted = append(deleted, opts1[i])
			i += 1
		} else if cmp > 0 {
			added = append(added, opts2[j])
			j += 1
		} else {
			if (opts1[i].Value == nil && opts2[j].Value != nil) ||
				(opts1[i].Value != nil && opts2[j].Value == nil) ||
				(opts1[i].Value != nil && opts2[j].Value != nil && !opts1[i].Value.Equals(opts2[j].Value)) {
				updated1 = append(updated1, opts1[i])
				updated2 = append(updated2, opts2[j])
			}
			i += 1
			j += 1
		}
	}
	if i < len(opts1) {
		deleted = append(deleted, opts1[i:]...)
	}
	if j < len(opts2) {
		added = append(added, opts2[j:]...)
	}
	return
}

func (manager *SConfigOptionManager) getDriver(idpId string) (string, error) {
	idp, _ := db.NewModelObject(IdentityProviderManager)
	idp.(*SIdentityProvider).Id = idpId
	opts, err := manager.fetchConfigs(idp, []string{"identity"}, []string{"driver"})
	if err != nil {
		return "", errors.Wrap(err, "WhitelistedConfigManager.fetchConfigs")
	}
	if len(opts) == 1 {
		return opts[0].Value.GetString()
	}
	return api.IdentityDriverSQL, nil
}

func GetConfigs(model db.IModel, sensitive bool, whiteList, blackList map[string][]string) (api.TConfigs, error) {
	opts, err := WhitelistedConfigManager.fetchConfigs(model, nil, nil)
	if err != nil {
		return nil, err
	}
	opts = filterOptions(opts, whiteList, blackList)
	if sensitive {
		opts2, err := SensitiveConfigManager.fetchConfigs(model, nil, nil)
		if err != nil {
			return nil, err
		}
		opts = append(opts, opts2...)
	}
	if len(opts) == 0 {
		return nil, nil
	}

	return config2map(opts), nil
}

func saveConfigs(userCred mcclient.TokenCredential, action string, model db.IModel, opts api.TConfigs, whiteList map[string][]string, blackList map[string][]string, sensitiveConfs map[string][]string) (bool, error) {
	var err error
	changed := make([]sChangeOption, 0)
	changedSensitive := make([]sChangeOption, 0)
	whiteListedOpts, sensitiveOpts := getConfigOptions(opts, model, sensitiveConfs)
	whiteListedOpts = filterOptions(whiteListedOpts, whiteList, blackList)
	if action == "update" {
		err = whiteListedOpts.Validate()
		if err != nil {
			return false, err
		}
		changed, err = WhitelistedConfigManager.updateConfigs(model, whiteListedOpts)
		if err != nil {
			return false, errors.Wrap(err, "WhitelistedConfigManager.updateConfig")
		}
		changedSensitive, err = SensitiveConfigManager.updateConfigs(model, sensitiveOpts)
		if err != nil {
			return false, errors.Wrap(err, "SensitiveConfigManager.updateConfig")
		}
	} else if action == "remove" {
		changed, err = WhitelistedConfigManager.removeConfigs(model, whiteListedOpts)
		if err != nil {
			return false, errors.Wrap(err, "WhitelistedConfigManager.updateConfig")
		}
		changedSensitive, err = SensitiveConfigManager.removeConfigs(model, sensitiveOpts)
		if err != nil {
			return false, errors.Wrap(err, "SensitiveConfigManager.updateConfig")
		}
	} else {
		changed, err = WhitelistedConfigManager.syncConfigs(model, whiteListedOpts)
		if err != nil {
			return false, errors.Wrap(err, "WhitelistedConfigManager.syncConfig")
		}
		changedSensitive, err = SensitiveConfigManager.syncConfigs(model, sensitiveOpts)
		if err != nil {
			return false, errors.Wrap(err, "SensitiveConfigManager.syncConfig")
		}
	}
	maskedValue := jsonutils.NewString("*")
	for i := range changedSensitive {
		if changedSensitive[i].OValue != nil {
			changedSensitive[i].OValue = maskedValue
		}
		if changedSensitive[i].Value != nil {
			changedSensitive[i].Value = maskedValue
		}
	}
	changed = append(changed, changedSensitive...)
	if len(changed) > 0 {
		notes := jsonutils.Marshal(changed)
		if userCred == nil {
			userCred = getDefaultAdminCred()
		} else {
			logclient.AddSimpleActionLog(model, logclient.ACT_CHANGE_CONFIG, notes, userCred, true)
		}
		db.OpsLog.LogEvent(model, db.ACT_CHANGE_CONFIG, notes, userCred)
	}
	return len(changed) > 0, nil
}

type dbServiceConfigSession struct {
	config  *jsonutils.JSONDict
	service *SService

	commonService *SService
}

func NewServiceConfigSession() common_options.IServiceConfigSession {
	return &dbServiceConfigSession{}
}

func (s *dbServiceConfigSession) Merge(opts interface{}, serviceType string, serviceVersion string) bool {
	merged := false
	s.config = jsonutils.Marshal(opts).(*jsonutils.JSONDict)
	s.service, _ = ServiceManager.fetchServiceByType(serviceType)
	if s.service != nil {
		serviceConf, err := GetConfigs(s.service, false, nil, nil)
		if err != nil {
			log.Errorf("GetConfigs for %s fail: %s", serviceType, err)
		} else if serviceConf != nil {
			serviceConfJson := jsonutils.Marshal(serviceConf["default"])
			s.config.Update(serviceConfJson)
			merged = true
		} else {
			// not initialized
			// uploadConfig(s.service, s.config)
		}
	}
	s.commonService, _ = ServiceManager.fetchServiceByType(consts.COMMON_SERVICE)
	if s.commonService != nil {
		commonConf, err := GetConfigs(s.commonService, false, nil, nil)
		if err != nil {
			log.Errorf("GetConfigs for %s fail: %s", consts.COMMON_SERVICE, err)
		} else if commonConf != nil {
			commonConfJson := jsonutils.Marshal(commonConf["default"])
			s.config.Update(commonConfJson)
			merged = true
		} else {
			// common not initialized
			// uploadConfig(commonService, s.config)
		}
	}
	if merged {
		err := s.config.Unmarshal(opts)
		if err == nil {
			return true
		}
		log.Errorf("s.config.Unmarshal fail %s", err)
	}
	return false
}

func (s *dbServiceConfigSession) Upload() {
	if s.service == nil {
		return
	}
	uploadConfig(s.service, s.config)
	uploadConfig(s.commonService, s.config)
}

func (s *dbServiceConfigSession) IsRemote() bool {
	return false
}

func uploadConfig(service *SService, config jsonutils.JSONObject) {
	nconf := jsonutils.NewDict()
	nconf.Add(config, "default")
	tconf := api.TConfigs{}
	err := nconf.Unmarshal(tconf)
	if err != nil {
		log.Errorf("nconf.Unmarshal fail %s", err)
		return
	}
	if service.isCommonService() {
		_, err = saveConfigs(nil, "", service, tconf, api.CommonWhitelistOptionMap, nil, nil)
	} else {
		_, err = saveConfigs(nil, "", service, tconf, nil, api.MergeServiceConfigOptions(api.CommonWhitelistOptionMap, api.ServiceBlacklistOptionMap), nil)
	}
	if err != nil {
		log.Errorf("saveConfigs fail %s", err)
		return
	}
}