Cloudpods/backend/pkg/monitor/models/alertrecord.go

// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package models

import (
	"context"
	"database/sql"
	"strings"
	"time"

	"yunion.io/x/jsonutils"
	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"
	"yunion.io/x/pkg/util/rbacscope"
	"yunion.io/x/pkg/util/sets"
	"yunion.io/x/pkg/util/timeutils"
	"yunion.io/x/sqlchemy"

	"yunion.io/x/onecloud/pkg/apis/monitor"
	"yunion.io/x/onecloud/pkg/cloudcommon/db"
	"yunion.io/x/onecloud/pkg/httperrors"
	"yunion.io/x/onecloud/pkg/mcclient"
	"yunion.io/x/onecloud/pkg/util/filterclause"
	"yunion.io/x/onecloud/pkg/util/stringutils2"
)

var (
	AlertRecordManager *SAlertRecordManager
)

// +onecloud:swagger-gen-model-singular=alertrecord
// +onecloud:swagger-gen-model-plural=alertrecords
type SAlertRecordManager struct {
	db.SEnabledResourceBaseManager
	db.SStandaloneAnonResourceBaseManager
	SMonitorScopedResourceManager
}

type SAlertRecord struct {
	//db.SVirtualResourceBase
	db.SEnabledResourceBase
	db.SStandaloneAnonResourceBase
	SMonitorScopedResource

	AlertId   string               `width:"36" charset:"ascii" nullable:"false" list:"user" create:"required"`
	Level     string               `charset:"ascii" width:"36" nullable:"false" default:"normal" list:"user" update:"user"`
	State     string               `width:"36" charset:"ascii" nullable:"false" default:"unknown" list:"user" update:"user"`
	SendState string               `width:"36" charset:"ascii" default:"ok" list:"user" update:"user"`
	EvalData  jsonutils.JSONObject `list:"user" update:"user" length:"medium"`
	AlertRule jsonutils.JSONObject `list:"user" update:"user" length:"medium"`
	ResType   string               `width:"36" list:"user" update:"user"`
	ResIds    string               `length:"medium" list:"user"`
}

func init() {
	AlertRecordManager = &SAlertRecordManager{
		SStandaloneAnonResourceBaseManager: db.NewStandaloneAnonResourceBaseManager(
			SAlertRecord{},
			"alertrecord_tbl",
			"alertrecord",
			"alertrecords",
		),
	}

	AlertRecordManager.SetVirtualObject(AlertRecordManager)
}

func (manager *SAlertRecordManager) NamespaceScope() rbacscope.TRbacScope {
	return rbacscope.ScopeSystem
}

func (manager *SAlertRecordManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, keys stringutils2.SSortedStrings) (*sqlchemy.SQuery, error) {
	q, err := manager.SStandaloneAnonResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys)
	if err != nil {
		return nil, errors.Wrap(err, "SStatusStandaloneResourceBaseManager.ListItemExportKeys")
	}
	q, err = manager.SScopedResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys)
	if err != nil {
		return nil, errors.Wrap(err, "SScopedResourceBaseManager.ListItemExportKeys")
	}
	return q, nil
}

func (m *SAlertRecordManager) FilterByOwner(ctx context.Context, q *sqlchemy.SQuery, man db.FilterByOwnerProvider, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, scope rbacscope.TRbacScope) *sqlchemy.SQuery {
	return m.SMonitorScopedResourceManager.FilterByOwner(ctx, q, man, userCred, ownerId, scope)
}

func (manager *SAlertRecordManager) ListItemFilter(
	ctx context.Context, q *sqlchemy.SQuery,
	userCred mcclient.TokenCredential,
	query monitor.AlertRecordListInput,
) (*sqlchemy.SQuery, error) {
	var err error

	q, err = manager.SStandaloneAnonResourceBaseManager.ListItemFilter(ctx, q, userCred, query.StandaloneAnonResourceListInput)
	if err != nil {
		return nil, errors.Wrap(err, "SStandaloneResourceBaseManager.ListItemFilter")
	}
	q, err = manager.SEnabledResourceBaseManager.ListItemFilter(ctx, q, userCred,
		query.EnabledResourceBaseListInput)
	if err != nil {
		return nil, errors.Wrap(err, "SEnabledResourceBaseManager.ListItemFilter")
	}
	q, err = manager.SScopedResourceBaseManager.ListItemFilter(ctx, q, userCred,
		query.ScopedResourceBaseListInput)
	if err != nil {
		return nil, errors.Wrap(err, "SScopedResourceBaseManager.ListItemFilter")
	}
	if query.Alerting {
		alertingQuery := manager.getAlertingRecordQuery().SubQuery()

		q.Join(alertingQuery, sqlchemy.Equals(q.Field("alert_id"), alertingQuery.Field("alert_id"))).Filter(
			sqlchemy.Equals(q.Field("created_at"), alertingQuery.Field("max_created_at")))
	}
	if len(query.Level) != 0 {
		q.Filter(sqlchemy.Equals(q.Field("level"), query.Level))
	}
	if len(query.State) != 0 {
		q.Filter(sqlchemy.Equals(q.Field("state"), query.State))
	}
	if len(query.AlertId) != 0 {
		q.Filter(sqlchemy.Equals(q.Field("alert_id"), query.AlertId))
	} else {
		q = q.IsNotEmpty("res_type").IsNotNull("res_type")
	}
	if len(query.AlertName) != 0 {
		alertQ := GetCommonAlertManager().Query("id").Contains("name", query.AlertName).SubQuery()
		q = q.In("alert_id", alertQ)
	}
	if len(query.ResType) != 0 {
		q.Filter(sqlchemy.Equals(q.Field("res_type"), query.ResType))
	}
	if len(query.ResId) != 0 {
		q.Filter(sqlchemy.ContainsAny(q.Field("res_ids"), []string{query.ResId}))
	}
	if len(query.Filter) != 0 {
		for i, _ := range query.Filter {
			if strings.Contains(query.Filter[i], "trigger_time") {
				timeFilter := strings.ReplaceAll(query.Filter[i], "trigger_time", "created_at")
				filterClause := filterclause.ParseFilterClause(timeFilter)
				q.Filter(filterClause.QueryCondition(q))
			}
		}
	}
	return q, nil
}

func (man *SAlertRecordManager) getAlertingRecordQuery() *sqlchemy.SQuery {
	q := CommonAlertManager.Query("id").IsTrue("enabled").IsNull("used_by")
	q = q.Filter(sqlchemy.OR(sqlchemy.Equals(q.Field("state"), monitor.AlertStateAlerting),
		sqlchemy.Equals(q.Field("state"), monitor.AlertStatePending)))

	alertsQuery := q.SubQuery()
	recordSub := man.Query().SubQuery()

	recordQuery := recordSub.Query(recordSub.Field("alert_id"), sqlchemy.MAX("max_created_at", recordSub.Field("created_at")))
	recordQuery.Equals("state", monitor.AlertStateAlerting)
	recordQuery.In("alert_id", alertsQuery)
	recordQuery.IsNotNull("res_type").IsNotEmpty("res_type")
	recordQuery.GroupBy("alert_id")
	return recordQuery

}

func (man *SAlertRecordManager) CustomizeFilterList(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*db.CustomizeListFilters, error) {
	filters := db.NewCustomizeListFilters()

	input := new(monitor.AlertRecordListInput)
	if err := query.Unmarshal(input); err != nil {
		return nil, errors.Wrap(err, "AlertRecordManager.CustomizeFilterList.Unmarshal")
	}
	if input.ResName != "" {
		filters.Append(func(item jsonutils.JSONObject) (bool, error) {
			evalMatchs := make([]monitor.EvalMatch, 0)
			if item.Contains("eval_data") {
				err := item.Unmarshal(&evalMatchs, "eval_data")
				if err != nil {
					return false, errors.Wrap(err, "record Unmarshal evalMatchs error")
				}
				for _, match := range evalMatchs {
					for k, v := range match.Tags {
						if strings.Contains(k, "name") && strings.Contains(v, input.ResName) {
							return true, nil
						}
					}
				}
			}
			return false, nil
		})
	}
	return filters, nil
}

func (man *SAlertRecordManager) GetAlertRecord(id string) (*SAlertRecord, error) {
	obj, err := man.FetchById(id)
	if err != nil {
		return nil, err
	}
	return obj.(*SAlertRecord), nil
}

func (man *SAlertRecordManager) GetAlertRecordsByAlertId(id string) ([]SAlertRecord, error) {
	records := make([]SAlertRecord, 0)
	query := man.Query()
	query = query.Equals("alert_id", id)
	err := db.FetchModelObjects(man, query, &records)
	if err != nil {
		return nil, err
	}
	return records, nil
}

func (manager *SAlertRecordManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) {
	var err error
	if field == "res_type" {
		resTypeQuery := MetricMeasurementManager.Query("res_type").Distinct()
		return resTypeQuery, nil
	}
	q, err = manager.SStandaloneAnonResourceBaseManager.QueryDistinctExtraField(q, field)
	if err == nil {
		return q, nil
	}
	return q, httperrors.ErrNotFound
}

func (man *SAlertRecordManager) OrderByExtraFields(
	ctx context.Context,
	q *sqlchemy.SQuery,
	userCred mcclient.TokenCredential,
	input monitor.AlertRecordListInput,
) (*sqlchemy.SQuery, error) {
	/*var err error

	q, err = man.SStatusStandaloneResourceBaseManager.OrderByExtraFields(ctx, q, userCred, input.StatusStandaloneResourceListInput)
	if err != nil {
		return nil, errors.Wrap(err, "SStandaloneResourceBaseManager.OrderByExtraFields")
	}
	/*q, err = man.SScopedResourceBaseManager.OrderByExtraFields(ctx, q, userCred, input.ScopedResourceBaseListInput)
	if err != nil {
		return nil, errors.Wrap(err, "SScopedResourceBaseManager.OrderByExtraFields")
	}*/
	return q, nil
}

func (man *SAlertRecordManager) FetchCustomizeColumns(
	ctx context.Context,
	userCred mcclient.TokenCredential,
	query jsonutils.JSONObject,
	objs []interface{},
	fields stringutils2.SSortedStrings,
	isList bool,
) []monitor.AlertRecordDetails {
	rows := make([]monitor.AlertRecordDetails, len(objs))
	stdRows := man.SStandaloneAnonResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
	scopedRows := man.SScopedResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList)
	alertIds := sets.NewString()
	records := make([]*SAlertRecord, len(objs))
	for i := range rows {
		rows[i] = monitor.AlertRecordDetails{
			StandaloneAnonResourceDetails: stdRows[i],
			ScopedResourceBaseInfo:        scopedRows[i],
		}
		record := objs[i].(*SAlertRecord)
		alertIds.Insert(record.AlertId)
		records[i] = record
	}

	alerts := map[string]SCommonAlert{}
	err := db.FetchModelObjectsByIds(CommonAlertManager, "id", alertIds.List(), &alerts)
	if err != nil {
		log.Errorf("FetchModelObjectsByIds common alert error: %v", err)
		return rows
	}
	for i := range rows {
		if alert, ok := alerts[records[i].AlertId]; ok {
			rows[i].AlertName = alert.Name
			rows[i].TriggerTime = records[i].CreatedAt
		}
		evalMatches, err := records[i].GetEvalData()
		if err != nil {
			continue
		}
		for j := range evalMatches {
			evalMatches[j] = records[i].filterTags(evalMatches[j])
		}
		rows[i].ResNum = int64(len(evalMatches))
	}
	return rows
}

func (record *SAlertRecord) filterTags(match monitor.EvalMatch) monitor.EvalMatch {
	for key, _ := range match.Tags {
		if strings.HasSuffix(key, "_id") {
			delete(match.Tags, key)
		}
	}
	return match
}

func (man *SAlertRecordManager) ValidateCreateData(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, _ jsonutils.JSONObject, data monitor.AlertRecordCreateInput) (monitor.AlertRecordCreateInput, error) {
	return data, nil
}

func (record *SAlertRecord) GetEvalData() ([]monitor.EvalMatch, error) {
	ret := make([]monitor.EvalMatch, 0)
	if record.EvalData == nil {
		return ret, nil
	}
	if err := record.EvalData.Unmarshal(&ret); err != nil {
		return nil, errors.Wrap(err, "unmarshal evalMatchs error")
	}
	return ret, nil
}

func (record *SAlertRecord) CustomizeCreate(
	ctx context.Context, userCred mcclient.TokenCredential,
	ownerId mcclient.IIdentityProvider,
	query jsonutils.JSONObject,
	data jsonutils.JSONObject,
) error {
	/*err := record.SMonitorScopedResource.CustomizeCreate(ctx, userCred, ownerId, query, data)
	if err != nil {
		return err
	}*/
	alert, err := AlertManager.GetAlert(record.AlertId)
	if err != nil {
		return errors.Wrapf(err, "GetAlert %s", record.AlertId)
	}
	record.DomainId = alert.GetDomainId()
	record.ProjectId = alert.GetProjectId()
	obj, err := db.NewModelObject(AlertRecordManager)
	if err != nil {
		return errors.Wrapf(err, "NewModelObject %s", AlertRecordManager.Keyword())
	}
	q := AlertRecordManager.Query().Equals("alert_id", record.AlertId).Desc("created_at")
	if err := q.First(obj); err != nil {
		if errors.Cause(err) == sql.ErrNoRows {
			return nil
		} else {
			return errors.Wrapf(err, "Get latest alertrecord error by alertId:%s", record.AlertId)
		}
	}
	latestRecord := obj.(*SAlertRecord)
	if latestRecord.GetState() == monitor.AlertStateAlerting && record.GetState() == monitor.AlertStateOK {
		err := record.unionEvalMatch(latestRecord)
		if err != nil {
			return errors.Wrap(err, "unionEvalMatch error")
		}
	}
	// fill res_ids
	matches, err := record.GetEvalData()
	if err != nil {
		return errors.Wrap(err, "GetEvalData error")
	}
	resIds := sets.NewString()
	for _, match := range matches {
		for k, v := range match.Tags {
			switch k {
			case "vm_id":
				if record.ResType == monitor.METRIC_RES_TYPE_AGENT || record.ResType == monitor.METRIC_RES_TYPE_GUEST {
					resIds.Insert(v)
				}
			case "host_id":
				if record.ResType == monitor.METRIC_RES_TYPE_HOST {
					resIds.Insert(v)
				}
			case "cloudaccount_id":
				if record.ResType == monitor.METRIC_RES_TYPE_CLOUDACCOUNT {
					resIds.Insert(v)
				}
			case "id":
				resIds.Insert(v)
			}
		}
	}
	record.ResIds = strings.Join(resIds.List(), ",")
	return nil
}

func (record *SAlertRecord) unionEvalMatch(alertingRecord *SAlertRecord) error {
	matches, err := record.GetEvalData()
	if err != nil {
		return err
	}
	alertingMatches, err := alertingRecord.GetEvalData()
	if err != nil {
		return err
	}
	newEvalMatchs := make([]monitor.EvalMatch, 0)
getNewMatchTag:
	for i, _ := range matches {
		for j, _ := range alertingMatches {
			if matches[i].Tags["name"] == alertingMatches[j].Tags["name"] {
				newEvalMatchs = append(newEvalMatchs, matches[i])
				continue getNewMatchTag
			}
		}
	}
	record.EvalData = jsonutils.Marshal(&newEvalMatchs)
	return nil
}

func (record *SAlertRecord) PostCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) {
	record.SStandaloneAnonResourceBase.PostCreate(ctx, userCred, ownerId, query, data)
	err := MonitorResourceManager.UpdateMonitorResourceAttachJointByRecord(ctx, userCred, record)
	if err != nil {
		log.Errorf("UpdateMonitorResourceAttachJointByRecord error: %v", err)
	}
	if err := GetAlertResourceManager().ReconcileFromRecord(ctx, userCred, ownerId, record); err != nil {
		log.Errorf("Reconcile from alert record error: %v", err)
		return
	}

	// err = GetAlertResourceManager().NotifyAlertResourceCount(ctx)
	// if err != nil {
	//	log.Errorf("NotifyAlertResourceCount error: %v", err)
	//	return
	// }
}

func (record *SAlertRecord) GetState() monitor.AlertStateType {
	return monitor.AlertStateType(record.State)
}

func (manager *SAlertRecordManager) DeleteRecordsOfThirtyDaysAgo(ctx context.Context, userCred mcclient.TokenCredential,
	isStart bool) {
	records := make([]SAlertRecord, 0)
	query := manager.Query()
	query = query.LE("created_at", timeutils.MysqlTime(time.Now().Add(-time.Hour*24*30)))
	err := db.FetchModelObjects(manager, query, &records)
	if err != nil {
		log.Errorf("fetch records ofthirty days ago err:%v", err)
		return
	}
	for i, _ := range records {
		err := db.DeleteModel(ctx, userCred, &records[i])
		if err != nil {
			log.Errorf("delete expire record: %s err: %v", records[i].GetId(), err)
		}
	}
}

func (manager *SAlertRecordManager) GetPropertyTotalAlert(ctx context.Context, userCred mcclient.TokenCredential,
	query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
	input := new(monitor.AlertRecordListInput)
	err := query.Unmarshal(input)
	if err != nil {
		return nil, errors.Wrap(err, "Unmarshal AlertRecordListInput error")
	}
	alertRess, err := MonitorResourceAlertManager.GetNowAlertingAlerts(ctx, userCred, input)
	if err != nil {
		return nil, errors.Wrap(err, "getNowAlertingRecord error")
	}
	alertCountMap := jsonutils.NewDict()
	for _, res := range alertRess {
		var count int64 = 1
		if alertCountMap.Contains(res.ResType) {
			resTypeCount, _ := alertCountMap.Int(res.ResType)
			count = count + resTypeCount
		}
		alertCountMap.Set(res.ResType, jsonutils.NewInt(count))
	}
	return alertCountMap, nil
}

// 获取过去一天的报警历史分布
func (manager *SAlertRecordManager) GetPropertyHistoryAlert(
	ctx context.Context,
	userCred mcclient.TokenCredential,
	query jsonutils.JSONObject,
) (*monitor.AlertRecordHistoryAlert, error) {
	q := manager.Query().GE("created_at", time.Now().Add(-time.Hour*24)).IsNotEmpty("eval_data").NotEquals("state", monitor.AlertStateOK)
	alerts := []SAlertRecord{}
	err := q.All(&alerts)
	if err != nil {
		return nil, errors.Wrap(err, "q.All")
	}
	ret := map[string]map[string]map[string]int64{}
	domainIds, projectIds := []string{}, []string{}
	domainMap := map[string]string{}
	projectMap := map[string]string{}
	for _, alert := range alerts {
		if _, ok := ret[alert.DomainId]; !ok {
			ret[alert.DomainId] = map[string]map[string]int64{}
			domainIds = append(domainIds, alert.DomainId)
		}
		if _, ok := ret[alert.DomainId][alert.ProjectId]; !ok {
			ret[alert.DomainId][alert.ProjectId] = map[string]int64{}
			projectIds = append(projectIds, alert.ProjectId)
		}
		if _, ok := ret[alert.DomainId][alert.ProjectId][alert.ResType]; !ok {
			ret[alert.DomainId][alert.ProjectId][alert.ResType] = 0
		}
		eval := make([]monitor.EvalMatch, 0)
		err := alert.EvalData.Unmarshal(&eval)
		if err != nil {
			continue
		}
		ret[alert.DomainId][alert.ProjectId][alert.ResType] += int64(len(eval))
	}
	domains := []db.STenant{}
	err = db.TenantCacheManager.GetDomainQuery().In("id", domainIds).All(&domains)
	if err != nil {
		return nil, errors.Wrap(err, "GetDomainQuery.In.All")
	}
	for _, domain := range domains {
		domainMap[domain.Id] = domain.Name
	}

	projects := []db.STenant{}
	err = db.TenantCacheManager.GetTenantQuery().In("id", projectIds).All(&projects)
	if err != nil {
		return nil, errors.Wrap(err, "GetTenantQuery.In.All")
	}
	for _, project := range projects {
		projectMap[project.Id] = project.Name
	}
	result := &monitor.AlertRecordHistoryAlert{}
	for domainId, projects := range ret {
		for projectId, resTypes := range projects {
			for resType, alert := range resTypes {
				result.Data = append(result.Data, monitor.AlertRecordHistoryAlertData{
					ProjectId: projectId,
					Project:   projectMap[projectId],
					DomainId:  domainId,
					Domain:    domainMap[domainId],
					ResType:   resType,
					ResNum:    alert,
				})
			}
		}
	}
	return result, nil
}

// GetPropertyProjectAlertResourceCount 获取指定时间段内各项目下的报警资源数量
func (manager *SAlertRecordManager) GetPropertyProjectAlertResourceCount(
	ctx context.Context,
	userCred mcclient.TokenCredential,
	input monitor.ProjectAlertResourceCountInput,
) (*monitor.ProjectAlertResourceCount, error) {
	// 验证时间段参数
	if input.StartTime.IsZero() || input.EndTime.IsZero() {
		return nil, httperrors.NewInputParameterError("start_time and end_time must be specified")
	}
	if input.StartTime.After(input.EndTime) {
		return nil, httperrors.NewInputParameterError("start_time must be before end_time")
	}

	// 构建查询
	q := manager.Query()
	q = q.GE("created_at", input.StartTime).LE("created_at", input.EndTime)
	q = q.IsNotEmpty("res_ids")

	// 应用权限过滤
	scope := rbacscope.ScopeSystem
	if input.Scope != "" {
		scope = rbacscope.TRbacScope(input.Scope)
	}
	q = manager.SMonitorScopedResourceManager.FilterByOwner(ctx, q, manager, userCred, userCred, scope)

	// 如果指定了 ResType，添加过滤条件
	if input.ResType != "" {
		q = q.Equals("res_type", input.ResType)
	}

	// 如果指定了 AlertId，添加过滤条件
	if input.AlertId != "" {
		q = q.Equals("alert_id", input.AlertId)
	}

	// 执行查询获取所有记录
	alerts := make([]SAlertRecord, 0)
	err := q.All(&alerts)
	if err != nil {
		return nil, errors.Wrap(err, "query alert records")
	}

	// 按 scope 分组统计唯一资源数量
	// systemResourceSet = set of resource IDs (system scope)
	// domainResourceSet[domainId] = set of resource IDs (domain scope)
	// projectResourceSet[domainId][projectId] = set of resource IDs (project scope)
	systemResourceSet := sets.NewString()
	domainResourceSet := make(map[string]sets.String)
	projectResourceSet := make(map[string]map[string]sets.String)
	domainIds := sets.NewString()
	projectIds := sets.NewString()

	for _, alert := range alerts {
		if len(alert.ResIds) == 0 {
			continue
		}
		domainId := alert.DomainId
		projectId := alert.ProjectId

		// 解析 res_ids（逗号分隔）
		resIds := strings.Split(alert.ResIds, ",")
		for _, resId := range resIds {
			resId = strings.TrimSpace(resId)
			if len(resId) == 0 {
				continue
			}

			// 根据 domainId 和 projectId 判断 scope
			if domainId == "" && projectId == "" {
				// system scope
				systemResourceSet.Insert(resId)
			} else if domainId != "" && projectId == "" {
				// domain scope
				domainIds.Insert(domainId)
				if domainResourceSet[domainId] == nil {
					domainResourceSet[domainId] = sets.NewString()
				}
				domainResourceSet[domainId].Insert(resId)
			} else if domainId != "" && projectId != "" {
				// project scope
				domainIds.Insert(domainId)
				projectIds.Insert(projectId)
				if projectResourceSet[domainId] == nil {
					projectResourceSet[domainId] = make(map[string]sets.String)
				}
				if projectResourceSet[domainId][projectId] == nil {
					projectResourceSet[domainId][projectId] = sets.NewString()
				}
				projectResourceSet[domainId][projectId].Insert(resId)
			}
		}
	}

	// 获取项目和域的名称
	domainMap := make(map[string]string)
	if domainIds.Len() > 0 {
		domains := []db.STenant{}
		err = db.TenantCacheManager.GetDomainQuery().In("id", domainIds.List()).All(&domains)
		if err != nil {
			return nil, errors.Wrap(err, "GetDomainQuery.In.All")
		}
		for _, domain := range domains {
			domainMap[domain.Id] = domain.Name
		}
	}

	projectMap := make(map[string]string)
	if projectIds.Len() > 0 {
		projects := []db.STenant{}
		err = db.TenantCacheManager.GetTenantQuery().In("id", projectIds.List()).All(&projects)
		if err != nil {
			return nil, errors.Wrap(err, "GetTenantQuery.In.All")
		}
		for _, project := range projects {
			projectMap[project.Id] = project.Name
		}
	}

	// 构建返回结果
	result := &monitor.ProjectAlertResourceCount{
		Data: make([]monitor.ProjectAlertResourceCountData, 0),
	}

	// system scope
	if systemResourceSet.Len() > 0 {
		result.Data = append(result.Data, monitor.ProjectAlertResourceCountData{
			Scope:    string(rbacscope.ScopeSystem),
			ResCount: int64(systemResourceSet.Len()),
		})
	}

	// domain scope
	for domainId, resourceSet := range domainResourceSet {
		if resourceSet.Len() > 0 {
			result.Data = append(result.Data, monitor.ProjectAlertResourceCountData{
				Scope:    string(rbacscope.ScopeDomain),
				DomainId: domainId,
				Domain:   domainMap[domainId],
				ResCount: int64(resourceSet.Len()),
			})
		}
	}

	// project scope
	for domainId, projects := range projectResourceSet {
		for projectId, resourceSet := range projects {
			if resourceSet.Len() > 0 {
				result.Data = append(result.Data, monitor.ProjectAlertResourceCountData{
					Scope:     string(rbacscope.ScopeProject),
					DomainId:  domainId,
					Domain:    domainMap[domainId],
					ProjectId: projectId,
					Project:   projectMap[projectId],
					ResCount:  int64(resourceSet.Len()),
				})
			}
		}
	}

	return result, nil
}