// Copyright 2019 Yunion // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package models import ( "context" "time" "yunion.io/x/cloudmux/pkg/cloudprovider" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/rbacscope" "yunion.io/x/sqlchemy" "yunion.io/x/sqlchemy/backends/clickhouse" "yunion.io/x/onecloud/pkg/apis" api "yunion.io/x/onecloud/pkg/apis/cloudevent" "yunion.io/x/onecloud/pkg/cloudcommon/consts" "yunion.io/x/onecloud/pkg/cloudcommon/db" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/util/stringutils2" ) type SCloudeventManager struct { db.SModelBaseManager db.SDomainizedResourceBaseManager } var CloudeventManager *SCloudeventManager func InitCloudevent() { if consts.OpsLogWithClickhouse { CloudeventManager = &SCloudeventManager{ SModelBaseManager: db.NewModelBaseManagerWithDBName( SCloudevent{}, "cloudevents_tbl", "cloudevent", "cloudevents", db.ClickhouseDB, ), } col := CloudeventManager.TableSpec().ColumnSpec("created_at") if clickCol, ok := col.(clickhouse.IClickhouseColumnSpec); ok { clickCol.SetTTL(consts.SplitableMaxKeepMonths(), "MONTH") } } else { CloudeventManager = &SCloudeventManager{ SModelBaseManager: db.NewModelBaseManagerWithSplitable( SCloudevent{}, "cloudevents_tbl", "cloudevent", "cloudevents", "event_id", "created_at", consts.SplitableMaxDuration(), consts.SplitableMaxKeepMonths(), ), } } CloudeventManager.SetVirtualObject(CloudeventManager) } type SCloudevent struct { db.SModelBase db.SDomainizedResourceBase EventId int64 `primary:"true" auto_increment:"true" list:"user" clickhouse_partition_by:"toInt64(event_id/100000000000)` Name string `width:"128" charset:"utf8" nullable:"false" index:"true" list:"user"` Service string `width:"64" charset:"utf8" nullable:"true" list:"user"` ResourceType string `width:"64" charset:"utf8" nullable:"true" list:"user"` Action string `width:"64" charset:"utf8" nullable:"true" list:"user"` RequestId string `width:"128" charset:"utf8" nullable:"true" list:"user"` Request jsonutils.JSONObject `charset:"utf8" nullable:"true" list:"user"` Account string `width:"64" charset:"utf8" nullable:"true" list:"user"` Success bool `nullable:"false" list:"user"` CreatedAt time.Time `nullable:"false" created_at:"true" index:"true" get:"user" list:"user" clickhouse_ttl:"6m"` CloudproviderId string `width:"64" charset:"utf8" nullable:"true" list:"user"` Manager string `width:"128" charset:"utf8" nullable:"false" index:"true" list:"user"` Provider string `width:"64" charset:"ascii" nullable:"false" list:"user"` Brand string `width:"64" charset:"ascii" list:"domain"` } func (event *SCloudevent) BeforeInsert() { t := time.Now().UTC() event.EventId = db.CurrentTimestamp(t) } func (manager *SCloudeventManager) CreateByInsertOrUpdate() bool { return false } // 云平台操作日志列表 func (manager *SCloudeventManager) ListItemFilter( ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, input api.CloudeventListInput, ) (*sqlchemy.SQuery, error) { q, err := manager.SModelBaseManager.ListItemFilter(ctx, q, userCred, input.ModelBaseListInput) if err != nil { return nil, errors.Wrap(err, "SModelBaseManager.ListItemFilter") } q, err = manager.SDomainizedResourceBaseManager.ListItemFilter(ctx, q, userCred, input.DomainizedResourceListInput) if err != nil { return nil, errors.Wrap(err, "SDomainizedResourceBaseManager.ListItemFilter") } if len(input.Providers) > 0 { q = q.In("provider", input.Providers) } if len(input.Brands) > 0 { q = q.In("brand", input.Brands) } if len(input.Service) > 0 { q = q.In("service", input.Service) } if len(input.Manager) > 0 { q = q.In("manager", input.Manager) } if len(input.Account) > 0 { q = q.In("account", input.Account) } if len(input.Action) > 0 { q = q.In("action", input.Action) } if len(input.ResourceType) > 0 { q = q.In("resource_type", input.ResourceType) } if input.Success != nil { q = q.Equals("success", *input.Success) } if !input.Since.IsZero() { q = q.GT("created_at", input.Since) } if !input.Until.IsZero() { q = q.LE("created_at", input.Until) } return q, nil } func (manager *SCloudeventManager) FetchCustomizeColumns( ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, objs []interface{}, fields stringutils2.SSortedStrings, isList bool, ) []api.CloudeventDetails { rows := make([]api.CloudeventDetails, len(objs)) base := manager.SModelBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) domainRows := manager.SDomainizedResourceBaseManager.FetchCustomizeColumns(ctx, userCred, query, objs, fields, isList) for i := range rows { rows[i].ModelBaseDetails = base[i] rows[i].DomainizedResourceInfo = domainRows[i] } return rows } func (self *SCloudevent) CustomizeCreate(ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) error { return self.SModelBase.CustomizeCreate(ctx, userCred, ownerId, query, data) } func (manager *SCloudeventManager) NamespaceScope() rbacscope.TRbacScope { return rbacscope.ScopeDomain } func (manager *SCloudeventManager) ResourceScope() rbacscope.TRbacScope { return rbacscope.ScopeDomain } func (self *SCloudevent) GetOwnerId() mcclient.IIdentityProvider { owner := db.SOwnerId{DomainId: self.DomainId} return &owner } func (manager *SCloudeventManager) FilterByOwner(ctx context.Context, q *sqlchemy.SQuery, man db.FilterByOwnerProvider, userCred mcclient.TokenCredential, owner mcclient.IIdentityProvider, scope rbacscope.TRbacScope) *sqlchemy.SQuery { return manager.SDomainizedResourceBaseManager.FilterByOwner(ctx, q, man, userCred, owner, scope) } func (manager *SCloudeventManager) FetchOwnerId(ctx context.Context, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) { return manager.SDomainizedResourceBaseManager.FetchOwnerId(ctx, data) } func (manager *SCloudeventManager) ListItemExportKeys(ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, keys stringutils2.SSortedStrings) (*sqlchemy.SQuery, error) { return manager.SDomainizedResourceBaseManager.ListItemExportKeys(ctx, q, userCred, keys) } func (manager *SCloudeventManager) QueryDistinctExtraField(q *sqlchemy.SQuery, field string) (*sqlchemy.SQuery, error) { return manager.SDomainizedResourceBaseManager.QueryDistinctExtraField(q, field) } func (manager *SCloudeventManager) OrderByExtraFields( ctx context.Context, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query api.CloudeventListInput, ) (*sqlchemy.SQuery, error) { return manager.SDomainizedResourceBaseManager.OrderByExtraFields(ctx, q, userCred, query.DomainizedResourceListInput) } func (manager *SCloudeventManager) SyncCloudevent(ctx context.Context, userCred mcclient.TokenCredential, cloudprovider *SCloudprovider, iEvents []cloudprovider.ICloudEvent) int { count := 0 for _, iEvent := range iEvents { event := &SCloudevent{ Name: iEvent.GetName(), Service: iEvent.GetService(), ResourceType: iEvent.GetResourceType(), Action: iEvent.GetAction(), Account: iEvent.GetAccount(), RequestId: iEvent.GetRequestId(), Request: iEvent.GetRequest(), Success: iEvent.IsSuccess(), Manager: cloudprovider.Name, Provider: cloudprovider.Provider, Brand: cloudprovider.Brand, CloudproviderId: cloudprovider.Id, } event.DomainId = cloudprovider.DomainId if len(event.Brand) == 0 { event.Brand = event.Provider } for k, v := range map[string]string{ "service": event.Service, "resoruce_type": event.ResourceType, "action": event.Action, "account": event.Account, "manager": event.Manager, "provider": event.Provider, "brand": event.Brand, } { db.DistinctFieldManager.InsertOrUpdate(ctx, manager, k, v) } event.CreatedAt = iEvent.GetCreatedAt() event.SetModelManager(manager, event) err := manager.TableSpec().Insert(ctx, event) if err != nil { log.Errorf("failed to insert event: %s for cloudprovider: %s(%s) error: %v", jsonutils.Marshal(event).PrettyString(), cloudprovider.Name, cloudprovider.Id, err) continue } count += 1 } return count } func (manager *SCloudeventManager) GetPagingConfig() *db.SPagingConfig { return &db.SPagingConfig{ Order: sqlchemy.SQL_ORDER_DESC, MarkerFields: []string{"created_at", "event_id"}, DefaultLimit: 20, } } func (manager *SCloudeventManager) GetPropertyDistinctField(ctx context.Context, userCred mcclient.TokenCredential, input apis.DistinctFieldInput) (jsonutils.JSONObject, error) { fields, err := db.DistinctFieldManager.GetObjectDistinctFields(manager.Keyword()) if err != nil { return nil, errors.Wrapf(err, "DistinctFieldManager.GetObjectDistinctFields") } fieldMaps := map[string][]string{} for _, field := range fields { _, ok := fieldMaps[field.Key] if !ok { fieldMaps[field.Key] = []string{} } fieldMaps[field.Key] = append(fieldMaps[field.Key], field.Value) } ret := map[string][]string{} for _, key := range input.Field { ret[key], _ = fieldMaps[key] } return jsonutils.Marshal(ret), nil }