// Copyright 2019 Yunion // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package db import ( "context" "database/sql" "fmt" "runtime/debug" "strings" "time" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/httputils" "yunion.io/x/sqlchemy" identityapi "yunion.io/x/onecloud/pkg/apis/identity" "yunion.io/x/onecloud/pkg/cloudcommon/consts" "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/mcclient" "yunion.io/x/onecloud/pkg/mcclient/auth" modules "yunion.io/x/onecloud/pkg/mcclient/modules/identity" "yunion.io/x/onecloud/pkg/util/stringutils2" "yunion.io/x/onecloud/pkg/util/tagutils" ) var ( DefaultProjectFetcher func(ctx context.Context, id string, domainId string) (*STenant, error) DefaultDomainFetcher func(ctx context.Context, id string) (*STenant, error) DefaultProjectsFetcher func(ctx context.Context, idList []string, isDomain bool) map[string]STenant DefaultDomainQuery func(fields ...string) *sqlchemy.SQuery DefaultProjectQuery func(fields ...string) *sqlchemy.SQuery ) type STenantCacheManager struct { SKeystoneCacheObjectManager } type STenant struct { SKeystoneCacheObject } func (t *STenant) GetTags() tagutils.TTagSet { objType := "project" if t.Domain == identityapi.KeystoneDomainRoot && t.DomainId == identityapi.KeystoneDomainRoot { objType = "domain" } tags, _ := Metadata.rawGetAll(objType, t.Id, nil, "") return tagutils.Map2Tagset(tags) } func NewTenant(idStr string, name string, domainId string, domainName string) STenant { return STenant{SKeystoneCacheObject: NewKeystoneCacheObject(idStr, name, domainId, domainName)} } func NewDomain(idStr, name string) STenant { return NewTenant(idStr, name, identityapi.KeystoneDomainRoot, identityapi.KeystoneDomainRoot) } func (tenant *STenant) GetModelManager() IModelManager { return TenantCacheManager } var TenantCacheManager *STenantCacheManager func init() { TenantCacheManager = &STenantCacheManager{ SKeystoneCacheObjectManager: NewKeystoneCacheObjectManager( STenant{}, "tenant_cache_tbl", "tenant_cache", "tenant_caches", )} // log.Debugf("Initialize tenant cache manager %s %s", TenantCacheManager.KeywordPlural(), TenantCacheManager) TenantCacheManager.SetVirtualObject(TenantCacheManager) DefaultProjectFetcher = TenantCacheManager.FetchTenantByIdOrNameInDomain DefaultDomainFetcher = TenantCacheManager.FetchDomainByIdOrName DefaultProjectsFetcher = fetchProjects DefaultDomainQuery = TenantCacheManager.GetDomainQuery DefaultProjectQuery = TenantCacheManager.GetTenantQuery } func RegistUserCredCacheUpdater() { auth.RegisterAuthHook(onAuthCompleteUpdateCache) } func onAuthCompleteUpdateCache(ctx context.Context, userCred mcclient.TokenCredential) { TenantCacheManager.updateTenantCache(ctx, userCred) UserCacheManager.updateUserCache(ctx, userCred) } func (manager *STenantCacheManager) InitializeData() error { q := manager.Query().IsNullOrEmpty("domain_id") tenants := make([]STenant, 0) err := FetchModelObjects(manager, q, &tenants) if err != nil && err != sql.ErrNoRows { return errors.Wrap(err, "query") } for i := range tenants { _, err := Update(&tenants[i], func() error { tenants[i].DomainId = identityapi.DEFAULT_DOMAIN_ID tenants[i].Domain = identityapi.DEFAULT_DOMAIN_NAME return nil }) if err != nil { return errors.Wrap(err, "update") } } return nil } func (manager *STenantCacheManager) updateTenantCache(ctx context.Context, userCred mcclient.TokenCredential) { item := SCachedTenant{ Id: userCred.GetProjectId(), Name: userCred.GetProjectName(), DomainId: userCred.GetProjectDomainId(), ProjectDomain: userCred.GetProjectDomain(), } manager.Save(ctx, item, false) } func (manager *STenantCacheManager) GetTenantQuery(fields ...string) *sqlchemy.SQuery { return manager.Query(fields...).NotEquals("domain_id", identityapi.KeystoneDomainRoot) } func (manager *STenantCacheManager) GetDomainQuery(fields ...string) *sqlchemy.SQuery { return manager.Query(fields...).Equals("domain_id", identityapi.KeystoneDomainRoot) } func (manager *STenantCacheManager) fetchTenant(ctx context.Context, idStr string, domainId string, isDomain bool, noExpireCheck bool, filter func(q *sqlchemy.SQuery) *sqlchemy.SQuery) (*STenant, error) { var q *sqlchemy.SQuery if isDomain { q = manager.GetDomainQuery() } else { q = manager.GetTenantQuery() if len(domainId) > 0 { q = q.Equals("domain_id", domainId) } } q = filter(q) caches := []STenant{} err := FetchModelObjects(manager, q, &caches) if err != nil { return nil, errors.Wrap(err, "FetchModelObjects") } normal := []STenant{} for _, cache := range caches { if noExpireCheck || !cache.IsExpired() { normal = append(normal, cache) } } if len(normal) > 1 { return nil, errors.Wrapf(httperrors.ErrDuplicateName, "duplicate tenant/domain name (%d)", len(normal)) } if len(normal) == 1 { return &normal[0], nil } if isDomain { return manager.fetchDomainFromKeystone(ctx, idStr) } else { return manager.fetchTenantFromKeystone(ctx, idStr, domainId) } } func (t *STenant) IsExpired() bool { if t.LastCheck.IsZero() { return true } now := time.Now().UTC() if t.LastCheck.Add(consts.GetTenantCacheExpireSeconds()).Before(now) { return true } return false } func (manager *STenantCacheManager) FetchTenantByIdOrNameInDomain(ctx context.Context, idStr string, domainId string) (*STenant, error) { if !stringutils2.IsUtf8(idStr) { t, err := manager.FetchTenantById(ctx, idStr) if t != nil { return t, nil } else if err != nil && errors.Cause(err) != sql.ErrNoRows { return nil, errors.Wrap(err, "FetchTenantById") } } return manager.fetchTenant(ctx, idStr, domainId, false, false, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { if stringutils2.IsUtf8(idStr) { return q.Equals("name", idStr) } else { return q.Filter(sqlchemy.OR( sqlchemy.Equals(q.Field("id"), idStr), sqlchemy.Equals(q.Field("name"), idStr), )) } }) } func (manager *STenantCacheManager) FetchTenantById(ctx context.Context, idStr string) (*STenant, error) { return manager.fetchTenantById(ctx, idStr, false) } func (manager *STenantCacheManager) FetchTenantByIdWithoutExpireCheck(ctx context.Context, idStr string) (*STenant, error) { // noExpireCheck should be true return manager.fetchTenantById(ctx, idStr, true) } func (manager *STenantCacheManager) fetchTenantById(ctx context.Context, idStr string, noExpireCheck bool) (*STenant, error) { return manager.fetchTenant(ctx, idStr, "", false, noExpireCheck, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { return q.Filter(sqlchemy.Equals(q.Field("id"), idStr)) }) } func (manager *STenantCacheManager) FetchTenantByNameInDomain(ctx context.Context, idStr string, domainId string) (*STenant, error) { return manager.fetchTenant(ctx, idStr, domainId, false, false, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { return q.Filter(sqlchemy.Equals(q.Field("name"), idStr)) }) } func (manager *STenantCacheManager) fetchTenantFromKeystone(ctx context.Context, idStr string, domainId string) (*STenant, error) { if len(idStr) == 0 { log.Debugf("fetch empty tenant!!!!\n%s", debug.Stack()) return nil, fmt.Errorf("Empty idStr") } // It is to query all domain's project. query := jsonutils.NewDict() query.Set("scope", jsonutils.NewString("system")) if len(domainId) > 0 { query.Set("domain_id", jsonutils.NewString(domainId)) } query.Set("pending_delete", jsonutils.NewString("all")) s := auth.GetAdminSession(ctx, consts.GetRegion()) tenant, err := modules.Projects.GetById(s, idStr, query) if err != nil { if je, ok := err.(*httputils.JSONClientError); ok && je.Code == 404 { return nil, sql.ErrNoRows } log.Errorf("fetch project %s fail %s", idStr, err) return nil, errors.Wrap(err, "modules.Projects.Get") } var item SCachedTenant err = tenant.Unmarshal(&item) if err != nil { return nil, errors.Wrapf(err, "Unmarshal %s", tenant) } return manager.Save(ctx, item, true) } func (manager *STenantCacheManager) FetchDomainByIdOrName(ctx context.Context, idStr string) (*STenant, error) { return manager.fetchTenant(ctx, idStr, "", true, false, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { if stringutils2.IsUtf8(idStr) { return q.Equals("name", idStr) } else { return q.Filter(sqlchemy.OR( sqlchemy.Equals(q.Field("id"), idStr), sqlchemy.Equals(q.Field("name"), idStr), )) } }) } func (manager *STenantCacheManager) FetchDomainById(ctx context.Context, idStr string) (*STenant, error) { return manager.fetchDomainById(ctx, idStr, false) } func (manager *STenantCacheManager) FetchDomainByIdWithoutExpireCheck(ctx context.Context, idStr string) (*STenant, error) { return manager.fetchDomainById(ctx, idStr, true) } func (manager *STenantCacheManager) fetchDomainById(ctx context.Context, idStr string, noExpireCheck bool) (*STenant, error) { return manager.fetchTenant(ctx, idStr, "", true, noExpireCheck, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { return q.Filter(sqlchemy.Equals(q.Field("id"), idStr)) }) } func (manager *STenantCacheManager) FetchDomainByName(ctx context.Context, idStr string) (*STenant, error) { return manager.fetchTenant(ctx, idStr, "", true, false, func(q *sqlchemy.SQuery) *sqlchemy.SQuery { return q.Filter(sqlchemy.Equals(q.Field("name"), idStr)) }) } func (manager *STenantCacheManager) fetchDomainFromKeystone(ctx context.Context, idStr string) (*STenant, error) { if len(idStr) == 0 { log.Debugf("fetch empty domain!!!!\n%s", debug.Stack()) return nil, fmt.Errorf("Empty idStr") } s := auth.GetAdminSession(ctx, consts.GetRegion()) tenant, err := modules.Domains.GetById(s, idStr, nil) if err != nil { if je, ok := err.(*httputils.JSONClientError); ok && je.Code == 404 { return nil, sql.ErrNoRows } log.Errorf("fetch project %s fail %s", idStr, err) return nil, errors.Wrap(err, "modules.Projects.Get") } var item SCachedTenant err = tenant.Unmarshal(&item) if err != nil { return nil, errors.Wrapf(err, "Unmarshal %s", tenant) } item.DomainId = identityapi.KeystoneDomainRoot item.ProjectDomain = identityapi.KeystoneDomainRoot return manager.Save(ctx, item, true) } func (manager *STenantCacheManager) Delete(ctx context.Context, idStr string) error { lockman.LockRawObject(ctx, manager.KeywordPlural(), idStr) defer lockman.ReleaseRawObject(ctx, manager.KeywordPlural(), idStr) objo, err := manager.FetchById(idStr) if err != nil && err != sql.ErrNoRows { log.Errorf("FetchTenantbyId fail %s", err) return err } if err == sql.ErrNoRows { return nil } return objo.Delete(ctx, nil) } func (manager *STenantCacheManager) Save(ctx context.Context, item SCachedTenant, saveMeta bool) (*STenant, error) { lockman.LockRawObject(ctx, manager.KeywordPlural(), item.Id) defer lockman.ReleaseRawObject(ctx, manager.KeywordPlural(), item.Id) objo, err := manager.FetchById(item.Id) if err != nil && err != sql.ErrNoRows { log.Errorf("FetchTenantbyId fail %s", err) return nil, errors.Wrapf(err, "TenantCache FetchById %s", item.Id) } // clean cache metadata if item.PendingDeleted && saveMeta { for k := range item.Metadata { if strings.HasPrefix(k, USER_TAG_PREFIX) { item.Metadata[k] = "none" } } } now := time.Now().UTC() if err == nil { obj := objo.(*STenant) if obj.Id == item.Id && obj.Name == item.Name && obj.Domain == item.ProjectDomain && obj.DomainId == item.DomainId { Update(obj, func() error { obj.LastCheck = now obj.PendingDeleted = item.PendingDeleted if obj.PendingDeleted { obj.PendingDeletedAt = item.PendingDeletedAt } return nil }) if saveMeta { Metadata.rawSetValues(ctx, item.objType(), item.Id, item.Metadata, true, "") } return obj, nil } _, err = Update(obj, func() error { obj.Id = item.Id obj.Name = item.Name obj.Domain = item.ProjectDomain obj.DomainId = item.DomainId obj.LastCheck = now obj.PendingDeleted = item.PendingDeleted if obj.PendingDeleted { obj.PendingDeletedAt = item.PendingDeletedAt } return nil }) if err != nil { return nil, err } else { if saveMeta { Metadata.rawSetValues(ctx, item.objType(), item.Id, item.Metadata, true, "") } return obj, nil } } else { objm, err := NewModelObject(manager) obj := objm.(*STenant) obj.Id = item.Id obj.Name = item.Name obj.Domain = item.ProjectDomain obj.DomainId = item.DomainId obj.LastCheck = now obj.PendingDeleted = item.PendingDeleted if obj.PendingDeleted { obj.PendingDeletedAt = item.PendingDeletedAt } err = manager.TableSpec().InsertOrUpdate(ctx, obj) if err != nil { return nil, errors.Wrap(err, "InsertOrUpdate") } else { if saveMeta { Metadata.rawSetValues(ctx, item.objType(), item.Id, item.Metadata, true, "") } return obj, nil } } } /*func (manager *STenantCacheManager) GenerateProjectUserCred(ctx context.Context, projectName string) (mcclient.TokenCredential, error) { project, err := manager.FetchTenantByIdOrName(ctx, projectName) if err != nil { return nil, err } return &mcclient.SSimpleToken{ Project: project.Name, ProjectId: project.Id, }, nil }*/ /*func (tenant *STenant) GetDomain() string { if len(tenant.Domain) == 0 { return identityapi.DEFAULT_DOMAIN_NAME } return tenant.Domain } func (tenant *STenant) GetDomainId() string { if len(tenant.DomainId) == 0 { return identityapi.DEFAULT_DOMAIN_ID } return tenant.DomainId }*/ func (manager *STenantCacheManager) findFirstProjectOfDomain(domainId string) (*STenant, error) { q := manager.Query().Equals("domain_id", domainId).Asc("created_at") tenant := STenant{} tenant.SetModelManager(manager, &tenant) err := q.First(&tenant) if err != nil { return nil, err } return &tenant, nil } func (manager *STenantCacheManager) fetchDomainTenantsFromKeystone(ctx context.Context, domainId string) error { if len(domainId) == 0 { log.Debugf("fetch empty domain!!!!") debug.PrintStack() return fmt.Errorf("Empty domainId") } s := auth.GetAdminSession(ctx, consts.GetRegion()) params := jsonutils.NewDict() params.Add(jsonutils.NewString(domainId), "domain_id") params.Add(jsonutils.JSONTrue, "details") params.Add(jsonutils.NewString("all"), "pending_delete") tenants, err := modules.Projects.List(s, params) if err != nil { return errors.Wrap(err, "Projects.List") } for _, tenant := range tenants.Data { var item SCachedTenant err := tenant.Unmarshal(&item) if err != nil { return errors.Wrapf(err, "Unmarshal %s", tenant) } _, err = manager.Save(ctx, item, true) if err != nil { return errors.Wrap(err, "save") } } return nil } func (manager *STenantCacheManager) FindFirstProjectOfDomain(ctx context.Context, domainId string) (*STenant, error) { tenant, err := manager.findFirstProjectOfDomain(domainId) if err != nil { if err == sql.ErrNoRows { err = manager.fetchDomainTenantsFromKeystone(ctx, domainId) if err != nil { return nil, errors.Wrap(err, "fetchDomainTenantsFromKeystone") } return manager.findFirstProjectOfDomain(domainId) } return nil, errors.Wrap(err, "findFirstProjectOfDomain.queryFirst") } return tenant, nil } func (tenant *STenant) GetProjectId() string { if tenant.IsDomain() { return "" } else { return tenant.Id } } func (tenant *STenant) GetTenantId() string { return tenant.GetProjectId() } func (tenant *STenant) GetProjectDomainId() string { if tenant.IsDomain() { return tenant.Id } else { return tenant.DomainId } } func (tenant *STenant) GetTenantName() string { return tenant.GetProjectName() } func (tenant *STenant) GetProjectName() string { if tenant.IsDomain() { return "" } else { return tenant.Name } } func (tenant *STenant) GetProjectDomain() string { if tenant.IsDomain() { return tenant.Name } else { return tenant.Domain } } func (tenant *STenant) GetUserId() string { return "" } func (tenant *STenant) GetUserName() string { return "" } func (tenant *STenant) GetDomainId() string { return "" } func (tenant *STenant) GetDomainName() string { return "" } func (tenant *STenant) IsDomain() bool { if tenant.DomainId == identityapi.KeystoneDomainRoot { return true } else { return false } } func (tenant *STenant) objType() string { if tenant.IsDomain() { return "domain" } else { return "project" } } func (tenant *STenant) GetAllClassMetadata() (map[string]string, error) { meta, err := Metadata.rawGetAll(tenant.objType(), tenant.GetId(), nil, CLASS_TAG_PREFIX) if err != nil { return nil, errors.Wrap(err, "rawGetAll") } ret := make(map[string]string) for k, v := range meta { if strings.HasPrefix(k, SYSTEM_ADMIN_PREFIX) { continue } ret[k[len(CLASS_TAG_PREFIX):]] = v } return ret, nil } func (manager *STenantCacheManager) ConvertIds(ids []string, isDomain bool) ([]string, error) { var q *sqlchemy.SQuery if isDomain { q = manager.GetDomainQuery("id") } else { q = manager.GetTenantQuery("id") } q = q.Filter(sqlchemy.OR( sqlchemy.In(q.Field("id"), stringutils2.RemoveUtf8Strings(ids)), sqlchemy.In(q.Field("name"), ids), )) q = q.Distinct() results := []struct { Id string }{} err := q.All(&results) if err != nil { return nil, errors.Wrap(err, "query") } ret := make([]string, len(results)) for i := range results { ret[i] = results[i].Id } return ret, nil }