Cloudpods/backend/pkg/cloudcommon/db/fetch.go

// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package db

import (
	"context"
	"database/sql"
	"fmt"
	"reflect"
	"strings"
	"time"

	"yunion.io/x/jsonutils"
	"yunion.io/x/pkg/errors"
	"yunion.io/x/pkg/gotypes"
	"yunion.io/x/pkg/util/rbacscope"
	"yunion.io/x/sqlchemy"

	"yunion.io/x/onecloud/pkg/cloudcommon/consts"
	"yunion.io/x/onecloud/pkg/cloudcommon/policy"
	"yunion.io/x/onecloud/pkg/httperrors"
	"yunion.io/x/onecloud/pkg/mcclient"
	"yunion.io/x/onecloud/pkg/util/rbacutils"
	"yunion.io/x/onecloud/pkg/util/stringutils2"
)

func FetchJointByIds(manager IJointModelManager, masterId, slaveId string, query jsonutils.JSONObject) (IJointModel, error) {
	obj, err := NewModelObject(manager)
	if err != nil {
		return nil, err
	}
	jointObj, ok := obj.(IJointModel)
	if !ok {
		return nil, fmt.Errorf("FetchByIds not a IJointModel")
	}
	q := manager.Query()
	masterField := q.Field(manager.GetIJointModelManager().GetMasterFieldName()) // queryField(q, manager.GetMasterManager())
	if masterField == nil {
		return nil, fmt.Errorf("cannot find master id")
	}
	slaveField := q.Field(manager.GetIJointModelManager().GetSlaveFieldName()) // queryField(q, manager.GetSlaveManager())
	if slaveField == nil {
		return nil, fmt.Errorf("cannot find slave id")
	}
	cond := sqlchemy.AND(sqlchemy.Equals(masterField, masterId), sqlchemy.Equals(slaveField, slaveId))
	q = q.Filter(cond)
	q = manager.FilterByParams(q, query)
	count, err := q.CountWithError()
	if err != nil {
		return nil, err
	}
	if count > 1 {
		return nil, sqlchemy.ErrDuplicateEntry
	} else if count == 0 {
		return nil, sql.ErrNoRows
	}
	err = q.First(jointObj)
	if err != nil {
		return nil, err
	}
	return jointObj, nil
}

func FetchById(manager IModelManager, idStr string) (IModel, error) {
	q := manager.Query()
	q = manager.FilterById(q, idStr)
	count, err := q.CountWithError()
	if err != nil {
		return nil, err
	}
	if count == 1 {
		obj, err := NewModelObject(manager)
		if err != nil {
			return nil, err
		}
		err = q.First(obj)
		if err != nil {
			return nil, err
		} else {
			return obj, nil
		}
	} else if count > 1 {
		return nil, sqlchemy.ErrDuplicateEntry
	} else {
		return nil, sql.ErrNoRows
	}
}

func FetchByName(ctx context.Context, manager IModelManager, userCred mcclient.IIdentityProvider, idStr string) (IModel, error) {
	q := manager.Query()
	q = manager.FilterByName(q, idStr)
	count, err := q.CountWithError()
	if err != nil {
		return nil, err
	}
	if count > 0 && userCred != nil {
		q = manager.FilterByOwner(ctx, q, manager, nil, userCred, manager.NamespaceScope())
		q = manager.FilterBySystemAttributes(q, nil, nil, manager.ResourceScope())
		count, err = q.CountWithError()
		if err != nil {
			return nil, err
		}
	}
	if count == 1 {
		obj, err := NewModelObject(manager)
		if err != nil {
			return nil, err
		}
		err = q.First(obj)
		if err != nil {
			return nil, err
		} else {
			return obj, nil
		}
	} else if count > 1 {
		return nil, sqlchemy.ErrDuplicateEntry
	} else {
		return nil, sql.ErrNoRows
	}
}

func FetchByIdOrName(ctx context.Context, manager IModelManager, userCred mcclient.IIdentityProvider, idStr string) (IModel, error) {
	if stringutils2.IsUtf8(idStr) {
		return FetchByName(ctx, manager, userCred, idStr)
	}
	obj, err := FetchById(manager, idStr)
	if err == sql.ErrNoRows {
		return FetchByName(ctx, manager, userCred, idStr)
	} else {
		return obj, err
	}
}

func isRawQuery(manager IModelManager, userCred mcclient.TokenCredential, query jsonutils.JSONObject, action string) bool {
	if query == nil || !query.Contains("delete") {
		return false
	}
	var useRawQuery bool
	// query senders are responsible for clear up other constraint
	// like setting "pendinge_delete" to "all"
	queryDelete, _ := query.GetString("delete")
	if queryDelete == "all" && policy.PolicyManager.Allow(rbacscope.ScopeSystem, userCred, consts.GetServiceType(), manager.KeywordPlural(), action).Result.IsAllow() {
		useRawQuery = true
	}
	return useRawQuery
}

func fetchItemById(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, idStr string, query jsonutils.JSONObject, useRawQuery bool) (IModel, error) {
	var q *sqlchemy.SQuery
	var err error
	if query != nil && !query.IsZero() {
		// if isListRbacAllowed(manager, userCred, true) {
		// 	query.(*jsonutils.JSONDict).Set("admin", jsonutils.JSONTrue)
		// }
		q = manager.NewQuery(ctx, userCred, query, useRawQuery)
		q, err = listItemQueryFilters(manager, ctx, q, userCred, query, policy.PolicyActionGet, false)
		if err != nil {
			return nil, err
		}
	} else {
		q = manager.Query()
	}
	q = manager.FilterById(q, idStr)
	count, err := q.CountWithError()
	if err != nil {
		return nil, err
	}
	if count == 1 {
		item, err := NewModelObject(manager)
		if err != nil {
			return nil, err
		}
		err = q.First(item)
		if err != nil {
			return nil, err
		}
		return item, nil
	} else if count > 1 {
		return nil, sqlchemy.ErrDuplicateEntry
	} else {
		return nil, sql.ErrNoRows
	}
}

func fetchItemByName(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, idStr string, query jsonutils.JSONObject, useRawQuery bool) (IModel, error) {
	var q *sqlchemy.SQuery
	var err error
	if query != nil && !query.IsZero() {
		q = manager.NewQuery(ctx, userCred, query, useRawQuery)
		q, err = listItemQueryFilters(manager, ctx, q, userCred, query, policy.PolicyActionGet, false)
		if err != nil {
			return nil, err
		}
	} else {
		q = manager.Query()
	}
	q = manager.FilterByName(q, idStr)
	count, err := q.CountWithError()
	if err != nil {
		return nil, err
	}
	if count > 0 {
		if gotypes.IsNil(query) {
			query = jsonutils.NewDict()
		}
		ownerId, _, err, _ := FetchCheckQueryOwnerScope(ctx, userCred, query, manager, rbacutils.ActionGet, true)
		if err != nil {
			return nil, httperrors.NewGeneralError(err)
		}
		q = manager.FilterByOwner(ctx, q, manager, userCred, ownerId, manager.NamespaceScope())
		q = manager.FilterBySystemAttributes(q, nil, nil, manager.ResourceScope())
		count, err = q.CountWithError()
		if err != nil {
			return nil, err
		}
	}
	if count == 1 {
		item, err := NewModelObject(manager)
		if err != nil {
			return nil, err
		}
		err = q.First(item)
		if err != nil {
			return nil, err
		}
		return item, nil
	} else if count > 1 {
		return nil, sqlchemy.ErrDuplicateEntry
	} else {
		return nil, sql.ErrNoRows
	}
}

func fetchItem(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, idStr string, query jsonutils.JSONObject) (IModel, error) {
	useRawQuery := isRawQuery(manager, userCred, query, policy.PolicyActionGet)
	item, err := fetchItemById(manager, ctx, userCred, idStr, query, useRawQuery)
	if err != nil {
		item, err = fetchItemByName(manager, ctx, userCred, idStr, query, useRawQuery)
	}
	if err != nil {
		return nil, err
	}
	if err := CheckRecordChecksumConsistent(item); err != nil {
		return nil, err
	}
	return item, nil
}

func FetchUserInfo(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) {
	userStr, key := jsonutils.GetAnyString2(data, []string{
		"user_id",
		"user",
	})
	if len(userStr) > 0 {
		data.(*jsonutils.JSONDict).Remove(key)
		u, err := DefaultUserFetcher(ctx, userStr)
		if err != nil {
			if err == sql.ErrNoRows {
				return nil, httperrors.NewResourceNotFoundError2("user", userStr)
			}
			return nil, errors.Wrap(err, "UserCacheManager.FetchUserByIdOrName")
		}
		ownerId := SOwnerId{
			UserDomain:   u.Domain,
			UserDomainId: u.DomainId,
			UserId:       u.Id,
			User:         u.Name,
		}
		return &ownerId, nil
	}
	return FetchProjectInfo(ctx, data)
}

var (
	ProjectFetchKeys = []string{
		"project_id",
		"tenant_id",
		"project",
		"tenant",
	}
	DomainFetchKeys = []string{
		"project_domain_id",
		"domain_id",
		"project_domain",
		// "domain",
	}
)

func FetchProjectInfo(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) {
	tenantId, key := jsonutils.GetAnyString2(data, ProjectFetchKeys)
	if len(tenantId) > 0 {
		data.(*jsonutils.JSONDict).Remove(key)
		domainId, _ := jsonutils.GetAnyString2(data, DomainFetchKeys)
		t, err := DefaultProjectFetcher(ctx, tenantId, domainId)
		if err != nil {
			if err == sql.ErrNoRows {
				return nil, httperrors.NewResourceNotFoundError2("project", tenantId)
			}
			return nil, errors.Wrap(err, "TenantCacheManager.FetchTenantByIdOrName")
		}
		ownerId := SOwnerId{
			Domain:    t.Domain,
			DomainId:  t.DomainId,
			ProjectId: t.Id,
			Project:   t.Name,
		}
		data.(*jsonutils.JSONDict).Set("project", jsonutils.NewString(t.Id))
		// 当资源的域和归属云账号的域不同时，会导致查找不到该资源
		// data.(*jsonutils.JSONDict).Set("project_domain", jsonutils.NewString(t.DomainId))
		return &ownerId, nil
	}
	return FetchDomainInfo(ctx, data)
}

func FetchDomainInfo(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) {
	domainId, key := jsonutils.GetAnyString2(data, DomainFetchKeys)
	if len(domainId) > 0 {
		data.(*jsonutils.JSONDict).Remove(key)
		domain, err := DefaultDomainFetcher(ctx, domainId)
		if err != nil {
			if err == sql.ErrNoRows {
				return nil, httperrors.NewResourceNotFoundError2("domain", domainId)
			}
			return nil, httperrors.NewGeneralError(err)
		}
		owner := SOwnerId{DomainId: domain.Id, Domain: domain.Name}
		data.(*jsonutils.JSONDict).Set("project_domain", jsonutils.NewString(domain.Id))
		return &owner, nil
	}
	return nil, nil
}

type sUsageManager struct{}

func (m *sUsageManager) KeywordPlural() string {
	return "usages"
}

func (m *sUsageManager) ResourceScope() rbacscope.TRbacScope {
	return rbacscope.ScopeProject
}

func (m *sUsageManager) FetchOwnerId(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) {
	return FetchProjectInfo(ctx, data)
}

func FetchUsageOwnerScope(ctx context.Context, userCred mcclient.TokenCredential, data jsonutils.JSONObject) (mcclient.IIdentityProvider, rbacscope.TRbacScope, error, rbacutils.SPolicyResult) {
	return FetchCheckQueryOwnerScope(ctx, userCred, data, &sUsageManager{}, policy.PolicyActionGet, true)
}

type IScopedResourceManager interface {
	KeywordPlural() string
	ResourceScope() rbacscope.TRbacScope
	FetchOwnerId(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error)
}

func UsagePolicyCheck(userCred mcclient.TokenCredential, manager IScopedResourceManager, scope rbacscope.TRbacScope) rbacutils.SPolicyResult {
	allowScope, policyTagFilters := policy.PolicyManager.AllowScope(userCred, consts.GetServiceType(), manager.KeywordPlural(), policy.PolicyActionList)
	if scope.HigherThan(allowScope) {
		return rbacutils.SPolicyResult{Result: rbacutils.Deny}
	}
	return policyTagFilters
}

func FetchCheckQueryOwnerScope(
	ctx context.Context,
	userCred mcclient.TokenCredential,
	data jsonutils.JSONObject,
	manager IScopedResourceManager,
	action string,
	doCheckRbac bool,
) (mcclient.IIdentityProvider, rbacscope.TRbacScope, error, rbacutils.SPolicyResult) {
	var scope rbacscope.TRbacScope

	var allowScope rbacscope.TRbacScope
	var requireScope rbacscope.TRbacScope
	var queryScope rbacscope.TRbacScope
	var policyTagFilters rbacutils.SPolicyResult

	resScope := manager.ResourceScope()

	allowScope, policyTagFilters = policy.PolicyManager.AllowScope(userCred, consts.GetServiceType(), manager.KeywordPlural(), action)

	ownerId, err := manager.FetchOwnerId(ctx, data)
	if err != nil {
		return nil, queryScope, errors.Wrap(err, "FetchOwnerId"), policyTagFilters
	}
	if ownerId != nil {
		switch resScope {
		case rbacscope.ScopeProject, rbacscope.ScopeDomain:
			if len(ownerId.GetProjectId()) > 0 {
				queryScope = rbacscope.ScopeProject
				if ownerId.GetProjectId() == userCred.GetProjectId() {
					requireScope = rbacscope.ScopeProject
				} else if ownerId.GetProjectDomainId() == userCred.GetProjectDomainId() {
					requireScope = rbacscope.ScopeDomain
				} else {
					requireScope = rbacscope.ScopeSystem
				}
			} else if len(ownerId.GetProjectDomainId()) > 0 {
				queryScope = rbacscope.ScopeDomain
				if ownerId.GetProjectDomainId() == userCred.GetProjectDomainId() {
					requireScope = rbacscope.ScopeDomain
				} else {
					requireScope = rbacscope.ScopeSystem
				}
			}
		case rbacscope.ScopeUser:
			queryScope = rbacscope.ScopeUser
			if ownerId.GetUserId() == userCred.GetUserId() {
				requireScope = rbacscope.ScopeUser
			} else {
				requireScope = rbacscope.ScopeSystem
			}
		}
	} else {
		ownerId = userCred
		reqScopeStr, _ := data.GetString("scope")
		if len(reqScopeStr) > 0 {
			if reqScopeStr == "max" || reqScopeStr == "maxallowed" {
				queryScope = allowScope
			} else {
				queryScope = rbacscope.String2Scope(reqScopeStr)
			}
		} else if data.Contains("admin") {
			isAdmin := jsonutils.QueryBoolean(data, "admin", false)
			if isAdmin && allowScope.HigherThan(rbacscope.ScopeProject) {
				queryScope = allowScope
			}
		} else if action == policy.PolicyActionGet {
			queryScope = allowScope
		} else {
			queryScope = resScope
		}
		// if resScope.HigherThan(queryScope) {
		// 	queryScope = resScope
		// }
		requireScope = queryScope
	}
	if doCheckRbac && (requireScope.HigherThan(allowScope) || policyTagFilters.Result.IsDeny()) {
		return nil, scope, httperrors.NewForbiddenError("not enough privilege to do %s:%s:%s (require:%s,allow:%s,query:%s)",
			consts.GetServiceType(), manager.KeywordPlural(), action,
			requireScope, allowScope, queryScope), policyTagFilters
	}
	return ownerId, queryScope, nil, policyTagFilters
}

func mapKeys(idMap map[string]string) []string {
	keys := make([]string, len(idMap))
	idx := 0
	for k := range idMap {
		keys[idx] = k
		idx += 1
	}
	return keys
}

func FetchIdNameMap2(manager IStandaloneModelManager, ids []string) (map[string]string, error) {
	idMap := make(map[string]string, len(ids))
	for _, id := range ids {
		idMap[id] = ""
	}
	return FetchIdNameMap(manager, idMap)
}

func FetchIdNameMap(manager IStandaloneModelManager, idMap map[string]string) (map[string]string, error) {
	return FetchIdFieldMap(manager, "name", idMap)
}

func FetchIdFieldMap2(manager IStandaloneModelManager, field string, ids []string) (map[string]string, error) {
	idMap := make(map[string]string, len(ids))
	for _, id := range ids {
		idMap[id] = ""
	}
	return FetchIdFieldMap(manager, field, idMap)
}

func FetchIdFieldMap(manager IStandaloneModelManager, field string, idMap map[string]string) (map[string]string, error) {
	q := manager.Query("id", field).In("id", mapKeys(idMap))
	rows, err := q.Rows()
	if err != nil {
		if errors.Cause(err) == sql.ErrNoRows {
			return idMap, nil
		} else {
			return idMap, errors.Wrap(err, "Query")
		}
	}
	defer rows.Close()
	for rows.Next() {
		var id string
		var name string
		err := rows.Scan(&id, &name)
		if err != nil {
			return idMap, errors.Wrap(err, "rows.Scan")
		}
		idMap[id] = name
	}
	return idMap, nil
}

func FetchModelObjectsByIds(modelManager IModelManager, fieldName string, ids []string, targets interface{}) error {
	err := FetchQueryObjectsByIds(modelManager.Query(), fieldName, ids, targets)
	if err != nil {
		return errors.Wrap(err, "FetchQueryObjectsByIds")
	}
	// try to call model's SetModelManager
	targetValue := reflect.Indirect(reflect.ValueOf(targets))
	for _, key := range targetValue.MapKeys() {
		modelValueV := targetValue.MapIndex(key)
		if modelValueV.Kind() != reflect.Struct {
			break
		}
		newModelValue := reflect.New(modelValueV.Type()).Elem()
		newModelValue.Set(modelValueV)
		modelValue := newModelValue.Addr().Interface()
		if model, ok := modelValue.(IModel); ok {
			model.SetModelManager(modelManager, model)
			targetValue.SetMapIndex(key, reflect.Indirect(reflect.ValueOf(model)))
		}
	}
	return nil
}

func FetchQueryObjectsByIds(q *sqlchemy.SQuery, fieldName string, ids []string, targets interface{}) error {
	if len(ids) == 0 {
		return nil
	}

	targetValue := reflect.Indirect(reflect.ValueOf(targets))
	if targetValue.Kind() != reflect.Map {
		return errors.Wrap(httperrors.ErrBadRequest, "receiver should be a map")
	}

	isTargetSlice := false
	modelType := targetValue.Type().Elem()
	if modelType.Kind() == reflect.Slice {
		isTargetSlice = true
		modelType = modelType.Elem()
	}

	query := q.In(fieldName, ids)
	rows, err := query.Rows()
	if err != nil {
		if err == sql.ErrNoRows {
			return nil
		}
		return err
	}
	defer rows.Close()

	targetsValue := reflect.Indirect(reflect.ValueOf(targets))
	for rows.Next() {
		mMap, err := query.Row2Map(rows)
		if err != nil {
			return errors.Wrap(err, "query.Row2Map")
		}
		fieldValue := mMap[fieldName]
		m := reflect.New(modelType).Interface() // a pointer
		err = query.RowMap2Struct(mMap, m)
		if err != nil {
			return errors.Wrap(err, "query.RowMap2Struct")
		}
		keyValue := reflect.ValueOf(fieldValue)
		valValue := reflect.Indirect(reflect.ValueOf(m))
		if isTargetSlice {
			sliceValue := targetValue.MapIndex(keyValue)
			if !sliceValue.IsValid() {
				sliceValue = reflect.New(reflect.SliceOf(modelType)).Elem()
			}
			sliceValue = reflect.Append(sliceValue, valValue)
			targetsValue.SetMapIndex(keyValue, sliceValue)
		} else {
			targetsValue.SetMapIndex(keyValue, valValue)
		}
	}
	return nil
}

func FetchStandaloneObjectsByIds(modelManager IModelManager, ids []string, targets interface{}) error {
	return FetchModelObjectsByIds(modelManager, "id", ids, targets)
}

func FetchField(modelMan IModelManager, field string, qCallback func(q *sqlchemy.SQuery) *sqlchemy.SQuery) ([]string, error) {
	q := modelMan.Query(field)
	if qCallback != nil {
		q = qCallback(q)
	}
	return FetchIds(q)
}

func FetchIds(q *sqlchemy.SQuery) ([]string, error) {
	rows, err := q.Rows()
	if err != nil {
		if errors.Cause(err) == sql.ErrNoRows {
			return nil, nil
		}
		return nil, errors.Wrapf(err, "q.Rows")
	}
	defer rows.Close()

	values := []string{}
	for rows.Next() {
		var value sql.NullString
		err := rows.Scan(&value)
		if err != nil {
			return values, errors.Wrap(err, "rows.Scan")
		}
		if value.Valid {
			values = append(values, value.String)
		}
	}
	return values, nil
}

func FetchDistinctField(modelManager IModelManager, field string) ([]string, error) {
	return FetchField(modelManager, field, func(q *sqlchemy.SQuery) *sqlchemy.SQuery {
		return q.Distinct()
	})
}

func Purge(modelManager IModelManager, field string, ids []string, forceDelete bool) error {
	if len(ids) == 0 {
		return nil
	}

	var splitByLen = func(data []string, splitLen int) [][]string {
		var result [][]string
		for i := 0; i < len(data); i += splitLen {
			end := i + splitLen
			if end > len(data) {
				end = len(data)
			}
			result = append(result, data[i:end])
		}
		return result
	}

	var purge = func(ids []string) error {
		vars := []interface{}{}
		placeholders := make([]string, len(ids))
		for i := range placeholders {
			placeholders[i] = "?"
			vars = append(vars, ids[i])
		}
		placeholder := strings.Join(placeholders, ",")
		sql := fmt.Sprintf(
			"delete from %s where %s in (%s)",
			modelManager.TableSpec().Name(), field, placeholder,
		)

		if !forceDelete {
			sql = fmt.Sprintf(
				"update %s set deleted=1, deleted_at= ? where %s in (%s)",
				modelManager.TableSpec().Name(), field, placeholder,
			)
			vars = append([]interface{}{time.Now()}, vars...)
		}
		_, err := sqlchemy.GetDB().Exec(
			sql, vars...,
		)
		if err != nil {
			return errors.Wrapf(err, strings.ReplaceAll(sql, "?", "%s"), vars...)
		}
		return nil
	}

	idsArr := splitByLen(ids, 100)
	for i := range idsArr {
		err := purge(idsArr[i])
		if err != nil {
			return errors.Wrapf(err, "purge")
		}
	}
	return nil
}