db_dispatcher.go 64 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050
  1. // Copyright 2019 Yunion
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package db
  15. import (
  16. "context"
  17. "database/sql"
  18. "fmt"
  19. "net/http"
  20. "reflect"
  21. "strings"
  22. "yunion.io/x/jsonutils"
  23. "yunion.io/x/log"
  24. "yunion.io/x/pkg/errors"
  25. "yunion.io/x/pkg/gotypes"
  26. "yunion.io/x/pkg/util/printutils"
  27. "yunion.io/x/pkg/util/rbacscope"
  28. "yunion.io/x/pkg/util/version"
  29. "yunion.io/x/pkg/utils"
  30. "yunion.io/x/sqlchemy"
  31. "yunion.io/x/onecloud/pkg/appsrv"
  32. "yunion.io/x/onecloud/pkg/appsrv/dispatcher"
  33. "yunion.io/x/onecloud/pkg/cloudcommon/consts"
  34. "yunion.io/x/onecloud/pkg/cloudcommon/db/lockman"
  35. "yunion.io/x/onecloud/pkg/cloudcommon/policy"
  36. "yunion.io/x/onecloud/pkg/httperrors"
  37. "yunion.io/x/onecloud/pkg/mcclient"
  38. "yunion.io/x/onecloud/pkg/mcclient/auth"
  39. "yunion.io/x/onecloud/pkg/util/filterclause"
  40. "yunion.io/x/onecloud/pkg/util/logclient"
  41. "yunion.io/x/onecloud/pkg/util/rbacutils"
  42. "yunion.io/x/onecloud/pkg/util/stringutils2"
  43. "yunion.io/x/onecloud/pkg/util/tagutils"
  44. )
  45. type DBModelDispatcher struct {
  46. manager IModelManager
  47. }
  48. func NewModelHandler(manager IModelManager) *DBModelDispatcher {
  49. // registerModelManager(manager)
  50. return &DBModelDispatcher{manager: manager}
  51. }
  52. func (dispatcher *DBModelDispatcher) Keyword() string {
  53. return dispatcher.manager.Keyword()
  54. }
  55. func (dispatcher *DBModelDispatcher) KeywordPlural() string {
  56. return dispatcher.manager.KeywordPlural()
  57. }
  58. func (dispatcher *DBModelDispatcher) ContextKeywordPlurals() [][]string {
  59. ctxMans := dispatcher.manager.GetContextManagers()
  60. if ctxMans != nil {
  61. keys := make([][]string, len(ctxMans))
  62. for i := 0; i < len(ctxMans); i += 1 {
  63. keys[i] = make([]string, len(ctxMans[i]))
  64. for j := 0; j < len(ctxMans[i]); j += 1 {
  65. keys[i][j] = ctxMans[i][j].KeywordPlural()
  66. }
  67. }
  68. return keys
  69. }
  70. return nil
  71. }
  72. func (dispatcher *DBModelDispatcher) Filter(f appsrv.FilterHandler) appsrv.FilterHandler {
  73. return auth.AuthenticateWithDelayDecision(f, true)
  74. }
  75. func (dispatcher *DBModelDispatcher) CustomizeHandlerInfo(handler *appsrv.SHandlerInfo) {
  76. dispatcher.manager.CustomizeHandlerInfo(handler)
  77. }
  78. func fetchUserCredential(ctx context.Context) mcclient.TokenCredential {
  79. return policy.FetchUserCredential(ctx)
  80. }
  81. var (
  82. searchOps = map[string]string{
  83. "contains": "contains",
  84. "startswith": "startswith",
  85. "endswith": "endswith",
  86. "empty": "isnullorempty",
  87. }
  88. )
  89. func parseSearchFieldkey(key string) (string, string) {
  90. for op, fn := range searchOps {
  91. if strings.HasSuffix(key, "__"+op) {
  92. key = key[:len(key)-(2+len(op))]
  93. return key, fn
  94. } else if strings.HasSuffix(key, "__i"+op) {
  95. key = key[:len(key)-(3+len(op))]
  96. return key, fn
  97. }
  98. }
  99. return key, ""
  100. }
  101. func listItemsQueryByColumn(manager IModelManager, q *sqlchemy.SQuery, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (*sqlchemy.SQuery, error) {
  102. if query == nil {
  103. return q, nil
  104. }
  105. qdata, err := query.GetMap()
  106. if err != nil {
  107. return nil, errors.Wrapf(err, "query.GetMap %s", query.String())
  108. }
  109. listF := searchFields(manager, userCred)
  110. for key := range qdata {
  111. if !strings.HasPrefix(key, "@") {
  112. continue
  113. }
  114. fn, op := parseSearchFieldkey(key[1:])
  115. if listF.Contains(fn) {
  116. colSpec := manager.TableSpec().ColumnSpec(fn)
  117. if colSpec != nil {
  118. arrV := jsonutils.GetQueryStringArray(query, key)
  119. if len(op) > 0 {
  120. strV := strings.Join(arrV, ",")
  121. filter := fmt.Sprintf("%s.%s(%s)", fn, op, strV)
  122. fc := filterclause.ParseFilterClause(filter)
  123. if fc != nil {
  124. cond := fc.QueryCondition(q)
  125. if cond != nil {
  126. q = q.Filter(cond)
  127. }
  128. }
  129. } else if len(arrV) > 1 {
  130. for i := range arrV {
  131. arrV[i] = sqlchemy.GetStringValue(colSpec.ConvertFromString(arrV[i]))
  132. }
  133. q = q.In(fn, arrV)
  134. } else if len(arrV) == 1 {
  135. strV := colSpec.ConvertFromString(arrV[0])
  136. q = q.Equals(fn, strV)
  137. }
  138. }
  139. }
  140. }
  141. return q, nil
  142. }
  143. func applyListItemsSearchFilters(manager IModelManager, ctx context.Context, q *sqlchemy.SQuery,
  144. userCred mcclient.TokenCredential, likes []string) (*sqlchemy.SQuery, error) {
  145. conds := make([]sqlchemy.ICondition, 0)
  146. for _, like := range likes {
  147. like = strings.TrimSpace(like)
  148. if len(like) > 0 {
  149. isAscii := utils.IsAscii(like)
  150. for _, colName := range searchFields(manager, userCred) {
  151. colSpec := manager.TableSpec().ColumnSpec(colName)
  152. if colSpec != nil && colSpec.IsSearchable() && (!colSpec.IsAscii() || (colSpec.IsAscii() && isAscii)) {
  153. conds = append(conds, sqlchemy.Contains(q.Field(colName), like))
  154. }
  155. }
  156. extraConds := manager.ExtraSearchConditions(ctx, q, like)
  157. if len(extraConds) > 0 {
  158. conds = append(conds, extraConds...)
  159. }
  160. }
  161. }
  162. if len(conds) > 0 {
  163. q = q.Filter(sqlchemy.OR(conds...))
  164. }
  165. return q, nil
  166. }
  167. func ApplyListItemsGeneralFilters(manager IModelManager, q *sqlchemy.SQuery,
  168. userCred mcclient.TokenCredential, filters []string, filterAny bool) (*sqlchemy.SQuery, error) {
  169. conds := make([]sqlchemy.ICondition, 0)
  170. schFields := searchFields(manager, userCred) // only filter searchable fields
  171. for _, f := range filters {
  172. fc := filterclause.ParseFilterClause(f)
  173. if fc != nil {
  174. if schFields.Contains(fc.GetField()) {
  175. cond := fc.QueryCondition(q)
  176. if cond != nil {
  177. conds = append(conds, cond)
  178. }
  179. }
  180. }
  181. }
  182. if len(conds) > 0 {
  183. if filterAny {
  184. q = q.Filter(sqlchemy.OR(conds...))
  185. } else {
  186. q = q.Filter(sqlchemy.AND(conds...))
  187. }
  188. }
  189. return q, nil
  190. }
  191. func applyListItemsGeneralJointFilters(manager IModelManager, q *sqlchemy.SQuery,
  192. userCred mcclient.TokenCredential, jointFilters []string, filterAny bool) (*sqlchemy.SQuery, error) {
  193. for _, f := range jointFilters {
  194. jfc := filterclause.ParseJointFilterClause(f)
  195. if jfc != nil {
  196. jointModelManager := GetModelManager(jfc.GetJointModelName())
  197. if jointModelManager == nil {
  198. return nil, httperrors.NewResourceNotFoundError("invalid joint resources %s", jfc.GetJointModelName())
  199. }
  200. hasKey := false
  201. for _, colume := range manager.TableSpec().Columns() {
  202. if colume.Name() == jfc.OriginKey {
  203. hasKey = true
  204. }
  205. }
  206. if !hasKey {
  207. return q, httperrors.NewInputParameterError("invalid joint filter %s, because %s doesn't have %s field", f, manager.Keyword(), jfc.OriginKey)
  208. }
  209. schFields := searchFields(jointModelManager, userCred)
  210. if schFields.Contains(jfc.GetField()) {
  211. sq := jointModelManager.Query(jfc.RelatedKey)
  212. cond := jfc.GetJointFilter(sq)
  213. if cond != nil {
  214. sq = sq.Filter(cond)
  215. if filterAny {
  216. q = q.Filter(sqlchemy.OR(sqlchemy.In(q.Field(jfc.OriginKey), sq)))
  217. } else {
  218. q = q.Filter(sqlchemy.AND(sqlchemy.In(q.Field(jfc.OriginKey), sq)))
  219. }
  220. }
  221. }
  222. }
  223. }
  224. return q, nil
  225. }
  226. func ListItemQueryFilters(manager IModelManager,
  227. ctx context.Context,
  228. q *sqlchemy.SQuery,
  229. userCred mcclient.TokenCredential,
  230. query jsonutils.JSONObject,
  231. action string,
  232. ) (*sqlchemy.SQuery, error) {
  233. return listItemQueryFilters(manager, ctx, q, userCred, query, action, false)
  234. }
  235. func listItemQueryFiltersRaw(
  236. manager IModelManager,
  237. ctx context.Context, q *sqlchemy.SQuery,
  238. userCred mcclient.TokenCredential,
  239. query jsonutils.JSONObject,
  240. action string,
  241. doCheckRbac bool,
  242. useRawQuery bool,
  243. ) (*sqlchemy.SQuery, error) {
  244. ownerId, queryScope, err, policyTagFilters := FetchCheckQueryOwnerScope(ctx, userCred, query, manager, action, doCheckRbac)
  245. if err != nil {
  246. return nil, httperrors.NewGeneralError(err)
  247. }
  248. if !policyTagFilters.IsEmpty() {
  249. query.(*jsonutils.JSONDict).Update(policyTagFilters.Json())
  250. log.Debugf("policyTagFilers: %s", query)
  251. }
  252. if query.Contains("export_keys") {
  253. exportKeys, _ := query.GetString("export_keys")
  254. keys := stringutils2.NewSortedStrings(strings.Split(exportKeys, ","))
  255. q, err = manager.ListItemExportKeys(ctx, q, userCred, keys)
  256. if err != nil {
  257. return nil, errors.Wrap(err, "ListItemExportKeys")
  258. }
  259. }
  260. q, err = ListItemFilter(manager, ctx, q, userCred, query)
  261. if err != nil {
  262. return nil, errors.Wrap(err, "ListItemFilter")
  263. }
  264. // XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
  265. // TURN ON automatic filter by column name, ONLY if query key starts with @!!!!
  266. // example: @name=abc&@city=111
  267. // XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
  268. q, err = listItemsQueryByColumn(manager, q, userCred, query)
  269. if err != nil {
  270. return nil, errors.Wrap(err, "listItemsQueryByColumn")
  271. }
  272. searches := jsonutils.GetQueryStringArray(query, "search")
  273. if len(searches) > 0 {
  274. q, err = applyListItemsSearchFilters(manager, ctx, q, userCred, searches)
  275. if err != nil {
  276. return nil, errors.Wrap(err, "applyListItemsSearchFilters")
  277. }
  278. }
  279. filterAny, _ := query.Bool("filter_any")
  280. filters := jsonutils.GetQueryStringArray(query, "filter")
  281. if len(filters) > 0 {
  282. q, err = ApplyListItemsGeneralFilters(manager, q, userCred, filters, filterAny)
  283. if err != nil {
  284. return nil, errors.Wrap(err, "ApplyListItemsGeneralFilters")
  285. }
  286. }
  287. jointFilter := jsonutils.GetQueryStringArray(query, "joint_filter")
  288. if len(jointFilter) > 0 {
  289. q, err = applyListItemsGeneralJointFilters(manager, q, userCred, jointFilter, filterAny)
  290. if err != nil {
  291. return nil, errors.Wrap(err, "applyListItemsGeneralJointFilters")
  292. }
  293. }
  294. if !useRawQuery {
  295. // Specifically for joint resource, these filters will exclude
  296. // deleted resources by joining with master/slave tables
  297. q = manager.FilterByOwner(ctx, q, manager, userCred, ownerId, queryScope)
  298. q = manager.FilterBySystemAttributes(q, userCred, query, queryScope)
  299. q = manager.FilterByHiddenSystemAttributes(q, userCred, query, queryScope)
  300. }
  301. if isShowDetails(query) {
  302. managerVal := reflect.ValueOf(manager)
  303. fName := "ExtendListQuery"
  304. funcVal := managerVal.MethodByName(fName)
  305. if funcVal.IsValid() && !funcVal.IsNil() {
  306. oldq := q
  307. fields, _ := GetDetailFields(manager, userCred)
  308. for _, f := range fields {
  309. q = q.AppendField(q.Field(f).Label(f))
  310. }
  311. q, err = ExtendListQuery(manager, ctx, q, userCred, query)
  312. if err != nil {
  313. if errors.Cause(err) != MethodNotFoundError {
  314. return nil, errors.Wrap(err, "ExtendQuery")
  315. } else {
  316. // else ignore
  317. q = oldq
  318. }
  319. } else {
  320. // force query no details
  321. query.(*jsonutils.JSONDict).Set("details", jsonutils.JSONFalse)
  322. }
  323. }
  324. }
  325. return q, nil
  326. }
  327. func isShowDetails(query jsonutils.JSONObject) bool {
  328. showDetails := false
  329. showDetailsJson, _ := query.Get("details")
  330. if showDetailsJson != nil {
  331. showDetails, _ = showDetailsJson.Bool()
  332. } else {
  333. showDetails = true
  334. }
  335. return showDetails
  336. }
  337. func listItemQueryFilters(manager IModelManager,
  338. ctx context.Context, q *sqlchemy.SQuery,
  339. userCred mcclient.TokenCredential,
  340. query jsonutils.JSONObject,
  341. action string,
  342. doCheckRbac bool,
  343. ) (*sqlchemy.SQuery, error) {
  344. return listItemQueryFiltersRaw(manager, ctx, q, userCred, query, action, doCheckRbac, false)
  345. }
  346. func mergeFields(metaFields, queryFields []string, isSysAdmin bool) stringutils2.SSortedStrings {
  347. meta := stringutils2.NewSortedStrings(metaFields)
  348. if len(queryFields) == 0 {
  349. return meta
  350. }
  351. query := stringutils2.NewSortedStrings(queryFields)
  352. _, mAndQ, qNoM := stringutils2.Split(meta, query)
  353. if !isSysAdmin {
  354. return mAndQ
  355. }
  356. // only sysadmin can specify list Fields
  357. return stringutils2.Merge(mAndQ, qNoM)
  358. }
  359. func Query2List(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, q *sqlchemy.SQuery, query jsonutils.JSONObject, delayFetch bool) ([]jsonutils.JSONObject, error) {
  360. metaFields, excludeFields := listFields(manager, userCred)
  361. fieldFilter := jsonutils.GetQueryStringArray(query, "field")
  362. allowListResult := IsAllowList(rbacscope.ScopeSystem, userCred, manager)
  363. listF := mergeFields(metaFields, fieldFilter, allowListResult.Result.IsAllow())
  364. listExcludes, _, _ := stringutils2.Split(stringutils2.NewSortedStrings(excludeFields), listF)
  365. showDetails := isShowDetails(query)
  366. var items []interface{}
  367. extraResults := make([]*jsonutils.JSONDict, 0)
  368. rows, err := q.Rows()
  369. if err != nil && err != sql.ErrNoRows {
  370. return nil, err
  371. }
  372. defer rows.Close()
  373. var exportKeys stringutils2.SSortedStrings
  374. if query.Contains("export_keys") {
  375. exportKeyStr, _ := query.GetString("export_keys")
  376. exportKeys = stringutils2.NewSortedStrings(strings.Split(exportKeyStr, ","))
  377. }
  378. for rows.Next() {
  379. item, err := NewModelObject(manager)
  380. if err != nil {
  381. return nil, err
  382. }
  383. if len(exportKeys) > 0 {
  384. rowMap, err := q.Row2Map(rows)
  385. if err != nil {
  386. return nil, err
  387. }
  388. extraData := jsonutils.NewDict()
  389. for k, v := range rowMap {
  390. if len(v) > 0 {
  391. extraData.Add(jsonutils.NewString(v), k)
  392. }
  393. }
  394. extraKeys := manager.GetExportExtraKeys(ctx, exportKeys, rowMap)
  395. if extraKeys != nil {
  396. extraData.Update(extraKeys)
  397. }
  398. err = q.RowMap2Struct(rowMap, item)
  399. if err != nil {
  400. return nil, err
  401. }
  402. extraResults = append(extraResults, extraData)
  403. } else {
  404. err = q.Row2Struct(rows, item)
  405. if err != nil {
  406. return nil, err
  407. }
  408. if delayFetch {
  409. err = Fetch(item)
  410. if err != nil {
  411. return nil, err
  412. }
  413. }
  414. }
  415. if err := CheckRecordChecksumConsistent(item); err != nil {
  416. return nil, err
  417. }
  418. items = append(items, item)
  419. }
  420. results := make([]*jsonutils.JSONDict, len(items))
  421. if showDetails || len(exportKeys) > 0 {
  422. var sortedListFields stringutils2.SSortedStrings
  423. if len(exportKeys) > 0 {
  424. sortedListFields = exportKeys
  425. } else if len(fieldFilter) > 0 {
  426. sortedListFields = stringutils2.NewSortedStrings(fieldFilter)
  427. }
  428. extraRows, err := FetchCustomizeColumns(manager, ctx, userCred, query, items, sortedListFields, true)
  429. if err != nil {
  430. return nil, errors.Wrap(err, "FetchCustomizeColumns")
  431. }
  432. if len(extraRows) == len(results) {
  433. for i := range results {
  434. jsonDict := extraRows[i].CopyExcludes(listExcludes...)
  435. if i < len(extraResults) {
  436. extraResults[i].Update(jsonDict)
  437. jsonDict = extraResults[i]
  438. }
  439. if len(fieldFilter) > 0 {
  440. jsonDict = jsonDict.CopyIncludes(fieldFilter...)
  441. }
  442. results[i] = jsonDict
  443. }
  444. } else {
  445. return nil, httperrors.NewInternalServerError("FetchCustomizeColumns return incorrect number of results")
  446. }
  447. } else {
  448. for i := range items {
  449. jsonDict := jsonutils.Marshal(items[i]).(*jsonutils.JSONDict).CopyExcludes(listExcludes...)
  450. if len(fieldFilter) > 0 {
  451. jsonDict = jsonDict.CopyIncludes(fieldFilter...)
  452. }
  453. results[i] = jsonDict
  454. }
  455. }
  456. for i := range items {
  457. i18nDict := items[i].(IModel).GetI18N(ctx)
  458. if i18nDict != nil {
  459. jsonDict := results[i]
  460. jsonDict.Set("_i18n", i18nDict)
  461. }
  462. }
  463. r := make([]jsonutils.JSONObject, len(items))
  464. for i := range results {
  465. r[i] = results[i]
  466. }
  467. return r, nil
  468. }
  469. func fetchContextObjectsIds(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, ctxIds []dispatcher.SResourceContext, queryDict *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
  470. var err error
  471. for i := 0; i < len(ctxIds); i += 1 {
  472. queryDict, err = fetchContextObjectIds(manager, ctx, userCred, ctxIds[i], queryDict)
  473. if err != nil {
  474. return nil, err
  475. }
  476. }
  477. return queryDict, nil
  478. }
  479. func fetchContextObjectIds(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, ctxId dispatcher.SResourceContext, queryDict *jsonutils.JSONDict) (*jsonutils.JSONDict, error) {
  480. ctxObj, err := fetchContextObject(manager, ctx, userCred, ctxId)
  481. if err != nil {
  482. return nil, err
  483. }
  484. queryDict.Add(jsonutils.NewString(ctxObj.GetId()), fmt.Sprintf("%s_id", ctxObj.GetModelManager().Keyword()))
  485. if len(ctxObj.GetModelManager().Alias()) > 0 {
  486. queryDict.Add(jsonutils.NewString(ctxObj.GetId()), fmt.Sprintf("%s_id", ctxObj.GetModelManager().Alias()))
  487. }
  488. return queryDict, nil
  489. }
  490. func fetchContextObjects(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, ctxIds []dispatcher.SResourceContext) ([]IModel, error) {
  491. ctxObjs := make([]IModel, len(ctxIds))
  492. for i := 0; i < len(ctxIds); i += 1 {
  493. ctxObj, err := fetchContextObject(manager, ctx, userCred, ctxIds[i])
  494. if err != nil {
  495. return nil, err
  496. }
  497. ctxObjs[i] = ctxObj
  498. }
  499. return ctxObjs, nil
  500. }
  501. func fetchContextObject(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, ctxId dispatcher.SResourceContext) (IModel, error) {
  502. ctxMans := manager.GetContextManagers()
  503. if ctxMans == nil {
  504. return nil, httperrors.NewInternalServerError("No context manager")
  505. }
  506. for i := 0; i < len(ctxMans); i += 1 {
  507. for j := 0; j < len(ctxMans[i]); j += 1 {
  508. if ctxMans[i][j].KeywordPlural() == ctxId.Type {
  509. ctxObj, err := fetchItem(ctxMans[i][j], ctx, userCred, ctxId.Id, nil)
  510. if err != nil {
  511. if err == sql.ErrNoRows {
  512. return nil, httperrors.NewResourceNotFoundError2(ctxMans[i][j].Keyword(), ctxId.Id)
  513. } else {
  514. return nil, err
  515. }
  516. }
  517. return ctxObj, nil
  518. }
  519. }
  520. }
  521. return nil, httperrors.NewInternalServerError("No such context %s(%s)", ctxId.Type, ctxId.Id)
  522. }
  523. func ListItems(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, ctxIds []dispatcher.SResourceContext) (*printutils.ListResult, error) {
  524. // 获取常规参数
  525. var err error
  526. var maxLimit int64 = consts.GetMaxPagingLimit()
  527. limit, _ := query.Int("limit")
  528. forceNoPaging := jsonutils.QueryBoolean(query, "force_no_paging", false)
  529. offset, _ := query.Int("offset")
  530. pagingMarker, _ := query.GetString("paging_marker")
  531. pagingOrderStr, _ := query.GetString("paging_order")
  532. pagingOrder := sqlchemy.QueryOrderType(strings.ToUpper(pagingOrderStr))
  533. // export data only
  534. exportLimit, err := query.Int("export_limit")
  535. if query.Contains("export_keys") && err == nil {
  536. limit = exportLimit
  537. }
  538. var q *sqlchemy.SQuery
  539. useRawQuery := isRawQuery(manager, userCred, query, policy.PolicyActionList)
  540. queryDict, ok := query.(*jsonutils.JSONDict)
  541. if !ok {
  542. return nil, fmt.Errorf("invalid query format")
  543. }
  544. if len(ctxIds) > 0 {
  545. queryDict, err = fetchContextObjectsIds(manager, ctx, userCred, ctxIds, queryDict)
  546. if err != nil {
  547. return nil, err
  548. }
  549. }
  550. queryDict, err = manager.ValidateListConditions(ctx, userCred, queryDict)
  551. if err != nil {
  552. return nil, err
  553. }
  554. pagingConf := manager.GetPagingConfig()
  555. if pagingConf == nil {
  556. if limit <= 0 {
  557. limit = consts.GetDefaultPagingLimit()
  558. }
  559. } else {
  560. if limit <= 0 {
  561. limit = int64(pagingConf.DefaultLimit)
  562. }
  563. if pagingOrder != sqlchemy.SQL_ORDER_ASC && pagingOrder != sqlchemy.SQL_ORDER_DESC {
  564. pagingOrder = pagingConf.Order
  565. }
  566. }
  567. splitable := manager.GetSplitTable()
  568. if splitable != nil {
  569. // handle splitable query, query each subtable, then union results
  570. metas, err := splitable.GetTableMetas()
  571. if err != nil {
  572. return nil, errors.Wrap(err, "splitable.GetTableMetas")
  573. }
  574. var subqs []sqlchemy.IQuery
  575. for _, meta := range metas {
  576. ts := splitable.GetTableSpec(meta)
  577. subq := ts.Query()
  578. subq, err = listItemQueryFiltersRaw(manager, ctx, subq, userCred, queryDict.Copy(), policy.PolicyActionList, true, useRawQuery)
  579. if err != nil {
  580. return nil, errors.Wrap(err, "listItemQueryFiltersRaw")
  581. }
  582. if pagingConf != nil {
  583. if limit > 0 {
  584. subq = subq.Limit(int(limit) + 1)
  585. }
  586. if len(pagingMarker) > 0 {
  587. markers := decodePagingMarker(pagingMarker)
  588. for markerIdx, marker := range markers {
  589. if markerIdx < len(pagingConf.MarkerFields) {
  590. if pagingOrder == sqlchemy.SQL_ORDER_ASC {
  591. subq = subq.GE(pagingConf.MarkerFields[markerIdx], marker)
  592. } else {
  593. subq = subq.LE(pagingConf.MarkerFields[markerIdx], marker)
  594. }
  595. }
  596. }
  597. }
  598. for _, f := range pagingConf.MarkerFields {
  599. if pagingOrder == sqlchemy.SQL_ORDER_ASC {
  600. subq = subq.Asc(f)
  601. } else {
  602. subq = subq.Desc(f)
  603. }
  604. }
  605. }
  606. if limit > 0 {
  607. subq = subq.Limit(int(limit) + 1)
  608. }
  609. subqs = append(subqs, subq)
  610. }
  611. union, err := sqlchemy.UnionWithError(subqs...)
  612. if err != nil {
  613. if errors.Cause(err) == sql.ErrNoRows {
  614. emptyList := printutils.ListResult{Data: []jsonutils.JSONObject{}}
  615. return &emptyList, nil
  616. } else {
  617. return nil, errors.Wrap(err, "sqlchemy.UnionWithError")
  618. }
  619. }
  620. q = union.Query()
  621. } else {
  622. q = manager.NewQuery(ctx, userCred, queryDict, useRawQuery)
  623. }
  624. q, err = listItemQueryFiltersRaw(manager, ctx, q, userCred, queryDict, policy.PolicyActionList, true, useRawQuery)
  625. if err != nil {
  626. return nil, errors.Wrap(err, "listItemQueryFiltersRaw")
  627. }
  628. var totalCnt int
  629. var totalJson jsonutils.JSONObject
  630. if pagingConf == nil {
  631. summaryStats := jsonutils.QueryBoolean(query, "summary_stats", false)
  632. if summaryStats {
  633. // calculate total
  634. totalQ := q.CountQuery()
  635. totalCnt, totalJson, err = manager.CustomizedTotalCount(ctx, userCred, query, totalQ)
  636. if err != nil {
  637. return nil, errors.Wrap(err, "CustomizedTotalCount")
  638. }
  639. } else {
  640. totalCnt, err = q.CountWithError()
  641. if err != nil {
  642. return nil, errors.Wrap(err, "CountWithError")
  643. }
  644. }
  645. //log.Debugf("total count %d", totalCnt)
  646. if totalCnt == 0 {
  647. emptyList := printutils.ListResult{Data: []jsonutils.JSONObject{}}
  648. return &emptyList, nil
  649. }
  650. }
  651. if int64(totalCnt) > maxLimit && (limit <= 0 || limit > maxLimit) && !forceNoPaging {
  652. limit = maxLimit
  653. }
  654. // orders defined in pagingConf should have the highest priority
  655. if pagingConf != nil {
  656. for _, f := range pagingConf.MarkerFields {
  657. if pagingOrder == sqlchemy.SQL_ORDER_ASC {
  658. q = q.Asc(f)
  659. } else {
  660. q = q.Desc(f)
  661. }
  662. }
  663. }
  664. var primaryCol sqlchemy.IColumnSpec
  665. primaryCols := manager.TableSpec().PrimaryColumns()
  666. if len(primaryCols) == 1 {
  667. primaryCol = primaryCols[0]
  668. }
  669. orderBy := jsonutils.GetQueryStringArray(queryDict, "order_by")
  670. order := sqlchemy.SQL_ORDER_DESC
  671. orderStr, _ := queryDict.GetString("order")
  672. if orderStr == "asc" {
  673. order = sqlchemy.SQL_ORDER_ASC
  674. }
  675. orderQuery := query.(*jsonutils.JSONDict).Copy()
  676. for _, orderByField := range orderBy {
  677. if pagingConf != nil && utils.IsInStringArray(orderByField, pagingConf.MarkerFields) {
  678. // skip markerField in pagingConf
  679. continue
  680. }
  681. colSpec := manager.TableSpec().ColumnSpec(orderByField)
  682. if colSpec == nil {
  683. orderQuery.Set(fmt.Sprintf("order_by_%s", orderByField), jsonutils.NewString(string(order)))
  684. }
  685. }
  686. q, err = OrderByExtraFields(manager, ctx, q, userCred, orderQuery)
  687. if err != nil {
  688. return nil, errors.Wrap(err, "OrderByExtraFields")
  689. }
  690. if orderBy == nil {
  691. orderBy = []string{}
  692. }
  693. if !q.IsGroupBy() {
  694. if primaryCol != nil && primaryCol.IsNumeric() {
  695. orderBy = append(orderBy, primaryCol.Name())
  696. } else if manager.TableSpec().ColumnSpec("created_at") != nil {
  697. orderBy = append(orderBy, "created_at")
  698. if manager.TableSpec().ColumnSpec("name") != nil {
  699. orderBy = append(orderBy, "name")
  700. }
  701. if primaryCol != nil {
  702. orderBy = append(orderBy, primaryCol.Name())
  703. }
  704. }
  705. }
  706. for _, orderByField := range orderBy {
  707. if pagingConf != nil && utils.IsInStringArray(orderByField, pagingConf.MarkerFields) {
  708. // skip markerField in pagingConf
  709. continue
  710. }
  711. for _, field := range q.QueryFields() {
  712. if orderByField == field.Name() {
  713. if order == sqlchemy.SQL_ORDER_ASC {
  714. q = q.Asc(field)
  715. } else {
  716. q = q.Desc(field)
  717. }
  718. break
  719. }
  720. }
  721. }
  722. if forceNoPaging {
  723. limit = 0
  724. }
  725. if pagingConf != nil {
  726. if limit > 0 {
  727. q = q.Limit(int(limit) + 1)
  728. }
  729. if len(pagingMarker) > 0 {
  730. markers := decodePagingMarker(pagingMarker)
  731. for markerIdx, marker := range markers {
  732. if markerIdx < len(pagingConf.MarkerFields) {
  733. if pagingOrder == sqlchemy.SQL_ORDER_ASC {
  734. q = q.GE(pagingConf.MarkerFields[markerIdx], marker)
  735. } else {
  736. q = q.LE(pagingConf.MarkerFields[markerIdx], marker)
  737. }
  738. }
  739. }
  740. }
  741. retList, err := Query2List(manager, ctx, userCred, q, queryDict, false)
  742. if err != nil {
  743. return nil, httperrors.NewGeneralError(err)
  744. }
  745. nextMarkers := make([]string, 0)
  746. if limit > 0 && int64(len(retList)) > limit {
  747. for _, markerField := range pagingConf.MarkerFields {
  748. nextMarker, _ := retList[limit].GetString(markerField)
  749. nextMarkers = append(nextMarkers, nextMarker)
  750. }
  751. retList = retList[:limit]
  752. }
  753. nextMarker := encodePagingMarker(nextMarkers)
  754. retResult := printutils.ListResult{
  755. Data: retList, Limit: int(limit),
  756. NextMarker: nextMarker,
  757. MarkerField: strings.Join(pagingConf.MarkerFields, ","),
  758. MarkerOrder: string(pagingOrder),
  759. }
  760. return &retResult, nil
  761. }
  762. customizeFilters, err := manager.CustomizeFilterList(ctx, q, userCred, queryDict)
  763. if err != nil {
  764. return nil, errors.Wrap(err, "CustomizeFilterList")
  765. }
  766. delayFetch := false
  767. if customizeFilters.IsEmpty() {
  768. if limit > 0 {
  769. q = q.Limit(int(limit))
  770. }
  771. if offset > 0 {
  772. q = q.Offset(int(offset))
  773. if primaryCol != nil && !query.Contains("export_keys") && consts.QueryOffsetOptimization {
  774. q.AppendField(q.Field(primaryCol.Name())) // query primary key only
  775. delayFetch = true
  776. log.Debugf("apply queryOffsetOptimization")
  777. }
  778. }
  779. }
  780. retList, err := Query2List(manager, ctx, userCred, q, queryDict, delayFetch)
  781. if err != nil {
  782. return nil, httperrors.NewGeneralError(err)
  783. }
  784. retCount := len(retList)
  785. // apply customizeFilters
  786. retList, err = customizeFilters.DoApply(retList)
  787. if err != nil {
  788. return nil, httperrors.NewGeneralError(err)
  789. }
  790. if len(retList) != retCount {
  791. totalCnt = len(retList)
  792. }
  793. if query.Contains("export_keys") {
  794. retList = getExportCols(query, retList)
  795. }
  796. paginate := false
  797. if !customizeFilters.IsEmpty() {
  798. // query not use Limit and Offset, do manual pagination
  799. paginate = true
  800. }
  801. return calculateListResult(retList, totalCnt, totalJson, int(limit), int(offset), paginate), nil
  802. }
  803. // 构造list返回详情
  804. func calculateListResult(data []jsonutils.JSONObject, total int, totalJson jsonutils.JSONObject, limit, offset int, paginate bool) *printutils.ListResult {
  805. if paginate {
  806. // do offset first
  807. if offset > 0 {
  808. if total > offset {
  809. data = data[offset:]
  810. } else {
  811. data = []jsonutils.JSONObject{}
  812. }
  813. }
  814. // do limit
  815. if limit > 0 && total-offset > limit {
  816. data = data[:limit]
  817. } else {
  818. data = data[:total-offset]
  819. }
  820. }
  821. retResult := printutils.ListResult{
  822. Data: data,
  823. Total: total,
  824. Limit: limit,
  825. Offset: offset,
  826. Totals: totalJson,
  827. }
  828. return &retResult
  829. }
  830. func getExportCols(query jsonutils.JSONObject, retList []jsonutils.JSONObject) []jsonutils.JSONObject {
  831. var exportKeys stringutils2.SSortedStrings
  832. exportKeyStr, _ := query.GetString("export_keys")
  833. exportKeys = stringutils2.NewSortedStrings(strings.Split(exportKeyStr, ","))
  834. for i := range retList {
  835. if len(exportKeys) > 0 {
  836. retList[i] = retList[i].(*jsonutils.JSONDict).CopyIncludes(exportKeys...)
  837. }
  838. }
  839. return retList
  840. }
  841. func (dispatcher *DBModelDispatcher) List(ctx context.Context, query jsonutils.JSONObject, ctxIds []dispatcher.SResourceContext) (*printutils.ListResult, error) {
  842. // 获取用户信息
  843. userCred := fetchUserCredential(ctx)
  844. manager := dispatcher.manager.GetImmutableInstance(ctx, userCred, query)
  845. ctx = manager.PrepareQueryContext(ctx, userCred, query)
  846. // list详情
  847. items, err := ListItems(manager, ctx, userCred, query, ctxIds)
  848. if err != nil {
  849. return nil, httperrors.NewGeneralError(errors.Wrapf(err, "ListItems"))
  850. }
  851. if userCred != nil && userCred.HasSystemAdminPrivilege() && manager.ListSkipLog(ctx, userCred, query) {
  852. appParams := appsrv.AppContextGetParams(ctx)
  853. if appParams != nil {
  854. appParams.SkipLog = true
  855. }
  856. }
  857. return items, nil
  858. }
  859. func getModelItemDetails(manager IModelManager, item IModel, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, isHead bool) (jsonutils.JSONObject, error) {
  860. appParams := appsrv.AppContextGetParams(ctx)
  861. if appParams == nil && isHead {
  862. return nil, httperrors.NewInternalServerError("fail to get http response writer from context")
  863. }
  864. hdrs := item.GetExtraDetailsHeaders(ctx, userCred, query)
  865. for k, v := range hdrs {
  866. appParams.Response.Header().Add(k, v)
  867. }
  868. if isHead {
  869. appParams.Response.Header().Add("Content-Length", "0")
  870. appParams.Response.Write([]byte{})
  871. return nil, nil
  872. }
  873. if manager.IsCustomizedGetDetailsBody() {
  874. return item.CustomizedGetDetailsBody(ctx, userCred, query)
  875. } else {
  876. return getItemDetails(manager, item, ctx, userCred, query)
  877. }
  878. }
  879. func GetItemDetails(manager IModelManager, item IModel, ctx context.Context, userCred mcclient.TokenCredential) (jsonutils.JSONObject, error) {
  880. return getItemDetails(manager, item, ctx, userCred, jsonutils.NewDict())
  881. }
  882. func getItemDetails(manager IModelManager, item IModel, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
  883. metaFields, excludeFields := GetDetailFields(manager, userCred)
  884. fieldFilter := jsonutils.GetQueryStringArray(query, "field")
  885. var sortedListFields stringutils2.SSortedStrings
  886. if len(fieldFilter) > 0 {
  887. sortedListFields = stringutils2.NewSortedStrings(fieldFilter)
  888. }
  889. extraRows, err := FetchCustomizeColumns(manager, ctx, userCred, query, []interface{}{item}, sortedListFields, false)
  890. if err != nil {
  891. return nil, errors.Wrap(err, "FetchCustomizeColumns")
  892. }
  893. if len(extraRows) == 1 {
  894. getFields := mergeFields(metaFields, fieldFilter, IsAllowGet(ctx, rbacscope.ScopeSystem, userCred, item))
  895. excludes, _, _ := stringutils2.Split(stringutils2.NewSortedStrings(excludeFields), getFields)
  896. return extraRows[0].CopyExcludes(excludes...), nil
  897. }
  898. return nil, httperrors.NewInternalServerError("FetchCustomizeColumns returns incorrect results(expect 1 actual %d)", len(extraRows))
  899. }
  900. func tryGetModelProperty(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, property string, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
  901. funcName := fmt.Sprintf("GetProperty%s", utils.Kebab2Camel(property, "-"))
  902. modelValue := reflect.ValueOf(manager)
  903. // params := []interface{}{ctx, userCred, query}
  904. funcValue := modelValue.MethodByName(funcName)
  905. if !funcValue.IsValid() || funcValue.IsNil() {
  906. return nil, nil
  907. }
  908. // _, _, err, _ := FetchCheckQueryOwnerScope(ctx, userCred, query, manager, policy.PolicyActionList, true)
  909. // if err != nil {
  910. // return nil, err
  911. // }
  912. outs, err := callFunc(funcValue, funcName, ctx, userCred, query)
  913. if err != nil {
  914. return nil, httperrors.NewInternalServerError("reflect call %s fail %s", funcName, err)
  915. }
  916. if len(outs) != 2 {
  917. return nil, httperrors.NewInternalServerError("Invald %s return value", funcName)
  918. }
  919. resVal := outs[0]
  920. errVal := outs[1].Interface()
  921. if !gotypes.IsNil(errVal) {
  922. return nil, errVal.(error)
  923. } else {
  924. if gotypes.IsNil(resVal) {
  925. return nil, httperrors.NewBadRequestError("No return value, so why query?")
  926. } else {
  927. return ValueToJSONObject(resVal), nil
  928. }
  929. }
  930. }
  931. func (dispatcher *DBModelDispatcher) Get(ctx context.Context, idStr string, query jsonutils.JSONObject, isHead bool) (jsonutils.JSONObject, error) {
  932. // log.Debugf("Get %s", idStr)
  933. userCred := fetchUserCredential(ctx)
  934. manager := dispatcher.manager.GetImmutableInstance(ctx, userCred, query)
  935. ctx = manager.PrepareQueryContext(ctx, userCred, query)
  936. data, err := tryGetModelProperty(manager, ctx, userCred, idStr, query)
  937. if err != nil {
  938. return nil, err
  939. } else if data != nil {
  940. if dataDict, ok := data.(*jsonutils.JSONDict); ok {
  941. i18nDict := manager.GetI18N(ctx, idStr, data)
  942. if i18nDict != nil {
  943. dataDict.Set("_i18n", i18nDict)
  944. }
  945. }
  946. return data, nil
  947. }
  948. model, err := fetchItem(manager, ctx, userCred, idStr, query)
  949. if err == sql.ErrNoRows {
  950. return nil, httperrors.NewResourceNotFoundError2(manager.Keyword(), idStr)
  951. } else if err != nil {
  952. return nil, errors.Wrapf(err, "fetchItem")
  953. }
  954. err = isObjectRbacAllowed(ctx, model, userCred, policy.PolicyActionGet)
  955. if err != nil {
  956. return nil, err
  957. }
  958. if userCred.HasSystemAdminPrivilege() && manager.GetSkipLog(ctx, userCred, query) {
  959. appParams := appsrv.AppContextGetParams(ctx)
  960. if appParams != nil {
  961. appParams.SkipLog = true
  962. }
  963. }
  964. return getModelItemDetails(manager, model, ctx, userCred, query, isHead)
  965. }
  966. func (dispatcher *DBModelDispatcher) GetSpecific(ctx context.Context, idStr string, spec string, query jsonutils.JSONObject) (jsonutils.JSONObject, error) {
  967. userCred := fetchUserCredential(ctx)
  968. manager := dispatcher.manager.GetImmutableInstance(ctx, userCred, query)
  969. ctx = manager.PrepareQueryContext(ctx, userCred, query)
  970. model, err := fetchItem(manager, ctx, userCred, idStr, query)
  971. if err == sql.ErrNoRows {
  972. return nil, httperrors.NewResourceNotFoundError2(manager.Keyword(), idStr)
  973. } else if err != nil {
  974. return nil, err
  975. }
  976. params := []interface{}{ctx, userCred, query}
  977. specCamel := utils.Kebab2Camel(spec, "-")
  978. modelValue := reflect.ValueOf(model)
  979. err = isObjectRbacAllowed(ctx, model, userCred, policy.PolicyActionGet, spec)
  980. if err != nil {
  981. return nil, err
  982. }
  983. funcName := fmt.Sprintf("GetDetails%s", specCamel)
  984. funcValue := modelValue.MethodByName(funcName)
  985. log.Errorf("MethodByName %s", funcName)
  986. if !funcValue.IsValid() || funcValue.IsNil() {
  987. log.Errorf("MethodByName2 %s", funcName)
  988. return nil, httperrors.NewSpecNotFoundError("%s %s %s not found", dispatcher.Keyword(), idStr, spec)
  989. }
  990. outs, err := callFunc(funcValue, funcName, params...)
  991. if err != nil {
  992. log.Errorf("MethodByName4 %s", funcName)
  993. return nil, err
  994. }
  995. if len(outs) != 2 {
  996. return nil, httperrors.NewInternalServerError("Invald %s return value", funcName)
  997. }
  998. resVal := outs[0]
  999. errVal := outs[1].Interface()
  1000. if !gotypes.IsNil(errVal) {
  1001. log.Errorf("MethodByName3 %s", funcName)
  1002. return nil, errVal.(error)
  1003. } else {
  1004. if gotypes.IsNil(resVal.Interface()) {
  1005. return nil, nil
  1006. } else {
  1007. return ValueToJSONObject(resVal), nil
  1008. }
  1009. }
  1010. }
  1011. func fetchOwnerId(ctx context.Context, manager IModelManager, userCred mcclient.TokenCredential, data jsonutils.JSONObject) (mcclient.IIdentityProvider, error) {
  1012. var ownerId mcclient.IIdentityProvider
  1013. var err error
  1014. if manager.ResourceScope() != rbacscope.ScopeSystem {
  1015. ownerId, err = manager.FetchOwnerId(ctx, data)
  1016. if err != nil {
  1017. return nil, httperrors.NewGeneralError(err)
  1018. }
  1019. }
  1020. if ownerId == nil {
  1021. ownerId = userCred
  1022. }
  1023. return ownerId, nil
  1024. }
  1025. /*func fetchOwnerProjectId(ctx context.Context, manager IModelManager, userCred mcclient.TokenCredential, data jsonutils.JSONObject) (string, error) {
  1026. var projId string
  1027. if data != nil {
  1028. // projId = jsonutils.GetAnyString(data, []string{"project", "tenant", "project_id", "tenant_id"})
  1029. projId = manager.FetchOwnerId(data)
  1030. }
  1031. ownerProjId := manager.GetOwnerId(userCred)
  1032. if len(projId) == 0 {
  1033. return ownerProjId, nil
  1034. }
  1035. tid, err := manager.ValidateOwnerId(ctx, projId)
  1036. if err != nil {
  1037. return "", httperrors.NewInputParameterError("Invalid owner %s", projId)
  1038. }
  1039. if tid == ownerProjId {
  1040. return ownerProjId, nil
  1041. }
  1042. var isAllow bool
  1043. if consts.IsRbacEnabled() {
  1044. result := policy.PolicyManager.Allow(true, userCred,
  1045. consts.GetServiceType(), policy.PolicyDelegation, "")
  1046. if result == rbacutils.AdminAllow {
  1047. isAllow = true
  1048. }
  1049. } else {
  1050. isAllow = userCred.IsAdminAllow(consts.GetServiceType(), policy.PolicyDelegation, "")
  1051. }
  1052. if !isAllow {
  1053. return "", httperrors.NewForbiddenError("Delegation not allowed")
  1054. }
  1055. return tid, nil
  1056. }*/
  1057. func NewModelObject(modelManager IModelManager) (IModel, error) {
  1058. m, ok := reflect.New(modelManager.TableSpec().DataType()).Interface().(IModel)
  1059. if !ok {
  1060. return nil, ErrInconsistentDataType
  1061. }
  1062. m.SetModelManager(modelManager, m)
  1063. return m, nil
  1064. }
  1065. func FetchModelObjects(modelManager IModelManager, query *sqlchemy.SQuery, targets interface{}) error {
  1066. rows, err := query.Rows()
  1067. if err != nil {
  1068. if err == sql.ErrNoRows {
  1069. return nil
  1070. }
  1071. return err
  1072. }
  1073. defer rows.Close()
  1074. targetsValue := reflect.Indirect(reflect.ValueOf(targets))
  1075. for rows.Next() {
  1076. m, err := NewModelObject(modelManager)
  1077. if err != nil {
  1078. return err
  1079. }
  1080. err = query.Row2Struct(rows, m)
  1081. if err != nil {
  1082. return err
  1083. }
  1084. newTargets := reflect.Append(targetsValue, reflect.Indirect(reflect.ValueOf(m)))
  1085. targetsValue.Set(newTargets)
  1086. }
  1087. return nil
  1088. }
  1089. func FetchIModelObjects(modelManager IModelManager, query *sqlchemy.SQuery) ([]IModel, error) {
  1090. // TODO: refactor below duplicated code from FetchModelObjects
  1091. rows, err := query.Rows()
  1092. if err != nil {
  1093. return nil, err
  1094. }
  1095. defer rows.Close()
  1096. objs := make([]IModel, 0)
  1097. for rows.Next() {
  1098. m, err := NewModelObject(modelManager)
  1099. if err != nil {
  1100. return nil, err
  1101. }
  1102. err = query.Row2Struct(rows, m)
  1103. if err != nil {
  1104. return nil, err
  1105. }
  1106. objs = append(objs, m)
  1107. }
  1108. return objs, nil
  1109. }
  1110. func DoCreate(manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject, ownerId mcclient.IIdentityProvider) (IModel, error) {
  1111. // 锁住一类实例
  1112. lockman.LockClass(ctx, manager, GetLockClassKey(manager, ownerId))
  1113. defer lockman.ReleaseClass(ctx, manager, GetLockClassKey(manager, ownerId))
  1114. return doCreateItem(manager, ctx, userCred, ownerId, nil, data)
  1115. }
  1116. func doCreateItem(
  1117. manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential,
  1118. ownerId mcclient.IIdentityProvider, query jsonutils.JSONObject, data jsonutils.JSONObject) (IModel, error) {
  1119. return _doCreateItem(manager, ctx, userCred, ownerId, query, data, false, 1)
  1120. }
  1121. // 批量创建
  1122. func batchCreateDoCreateItem(
  1123. manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider,
  1124. query jsonutils.JSONObject, data jsonutils.JSONObject, baseIndex int) (IModel, error) {
  1125. return _doCreateItem(manager, ctx, userCred, ownerId, query, data, true, baseIndex)
  1126. }
  1127. // 对于modelManager的实际创建过程
  1128. func _doCreateItem(
  1129. manager IModelManager, ctx context.Context, userCred mcclient.TokenCredential, ownerId mcclient.IIdentityProvider,
  1130. query jsonutils.JSONObject, data jsonutils.JSONObject, batchCreate bool, baseIndex int) (IModel, error) {
  1131. dataDict, ok := data.(*jsonutils.JSONDict)
  1132. if !ok {
  1133. return nil, httperrors.NewGeneralError(fmt.Errorf("fail to decode json data %s", data))
  1134. }
  1135. var err error
  1136. var generateName string
  1137. // 若manager存在name字段且请求包含generate_name,则根据name从数据库中获取相同名称添加后缀
  1138. if manager.HasName() {
  1139. if dataDict.Contains("generate_name") {
  1140. generateName, _ = dataDict.GetString("generate_name")
  1141. if len(generateName) > 0 {
  1142. if manager.EnableGenerateName() {
  1143. lockman.LockRawObject(ctx, manager.Keyword(), "name")
  1144. defer lockman.ReleaseRawObject(ctx, manager.Keyword(), "name")
  1145. // if enable generateName, alway generate name
  1146. newName, err := GenerateName2(ctx, manager, ownerId, generateName, nil, baseIndex)
  1147. if err != nil {
  1148. return nil, errors.Wrap(err, "GenerateName2")
  1149. }
  1150. dataDict.Add(jsonutils.NewString(newName), "name")
  1151. } else {
  1152. // if no name but generate_name provided, use generate_name as name instead
  1153. oldName, _ := dataDict.GetString("name")
  1154. if len(oldName) == 0 {
  1155. dataDict.Add(jsonutils.NewString(generateName), "name")
  1156. }
  1157. }
  1158. }
  1159. // cleanup generate_name
  1160. dataDict.Remove("generate_name")
  1161. }
  1162. }
  1163. funcName := "ValidateCreateData"
  1164. if batchCreate {
  1165. funcName = "BatchCreateValidateCreateData"
  1166. }
  1167. // 校验创建请求入参
  1168. dataDict, err = ValidateCreateData(funcName, manager, ctx, userCred, ownerId, query, dataDict)
  1169. if err != nil {
  1170. return nil, err
  1171. }
  1172. // 若manager用于name字段,确保name唯一
  1173. if manager.HasName() {
  1174. // run name validation after validate create data
  1175. name, _ := dataDict.GetString("name")
  1176. if len(name) > 0 {
  1177. uniqValues := manager.FetchUniqValues(ctx, dataDict)
  1178. err = NewNameValidator(ctx, manager, ownerId, name, uniqValues)
  1179. if err != nil {
  1180. return nil, err
  1181. }
  1182. }
  1183. }
  1184. // 检查models定义中tag指定required
  1185. err = jsonutils.CheckRequiredFields(dataDict, createRequireFields(manager, userCred))
  1186. if err != nil {
  1187. return nil, httperrors.NewInputParameterError("%v", err)
  1188. }
  1189. // 初始化model
  1190. model, err := NewModelObject(manager)
  1191. if err != nil {
  1192. return nil, httperrors.NewGeneralError(err)
  1193. }
  1194. // 检查models定义中tag指定create
  1195. filterData := dataDict.CopyIncludes(createFields(manager, userCred)...)
  1196. err = filterData.Unmarshal(model)
  1197. if err != nil {
  1198. return nil, httperrors.NewGeneralError(err)
  1199. }
  1200. // 实际创建前钩子
  1201. err = model.CustomizeCreate(ctx, userCred, ownerId, query, dataDict)
  1202. if err != nil {
  1203. return nil, httperrors.NewGeneralError(err)
  1204. }
  1205. dryRun := struct {
  1206. DryRun bool
  1207. }{}
  1208. data.Unmarshal(&dryRun)
  1209. if dryRun.DryRun {
  1210. return model, nil
  1211. }
  1212. // 插入数据库记录
  1213. if manager.CreateByInsertOrUpdate() {
  1214. err = manager.TableSpec().InsertOrUpdate(ctx, model)
  1215. } else {
  1216. err = manager.TableSpec().Insert(ctx, model)
  1217. }
  1218. if err != nil {
  1219. return nil, httperrors.NewGeneralError(err)
  1220. }
  1221. if manager.HasName() {
  1222. // HACK: save generateName
  1223. if len(generateName) > 0 && manager.EnableGenerateName() {
  1224. if standaloneMode, ok := model.(IStandaloneModel); ok {
  1225. standaloneMode.SetMetadata(ctx, "generate_name", generateName, userCred)
  1226. }
  1227. }
  1228. }
  1229. // HACK: set data same as dataDict
  1230. data.(*jsonutils.JSONDict).Update(dataDict)
  1231. return model, nil
  1232. }
  1233. func (dispatcher *DBModelDispatcher) FetchCreateHeaderData(ctx context.Context, header http.Header) (jsonutils.JSONObject, error) {
  1234. return dispatcher.manager.FetchCreateHeaderData(ctx, header)
  1235. }
  1236. func (dispatcher *DBModelDispatcher) Create(ctx context.Context, query jsonutils.JSONObject, data jsonutils.JSONObject, ctxIds []dispatcher.SResourceContext) (jsonutils.JSONObject, error) {
  1237. // 获取用户信息
  1238. userCred := fetchUserCredential(ctx)
  1239. manager := dispatcher.manager.GetMutableInstance(ctx, userCred, query, data)
  1240. ownerId, err := fetchOwnerId(ctx, manager, userCred, data)
  1241. if err != nil {
  1242. return nil, httperrors.NewGeneralError(err)
  1243. }
  1244. if len(ctxIds) > 0 {
  1245. dataDict, ok := data.(*jsonutils.JSONDict)
  1246. if !ok {
  1247. return nil, httperrors.NewGeneralError(fmt.Errorf("fail to parse body %s", data))
  1248. }
  1249. data, err = fetchContextObjectsIds(manager, ctx, userCred, ctxIds, dataDict)
  1250. if err != nil {
  1251. return nil, httperrors.NewGeneralError(errors.Wrapf(err, "fetchContextObjectsIds"))
  1252. }
  1253. }
  1254. // 用户角色校验
  1255. var policyResult rbacutils.SPolicyResult
  1256. policyResult, err = isClassRbacAllowed(ctx, manager, userCred, ownerId, policy.PolicyActionCreate)
  1257. if err != nil {
  1258. return nil, errors.Wrap(err, "isClassRbacAllowed")
  1259. }
  1260. // initialize pending usage in context any way
  1261. if InitPendingUsagesInContext != nil {
  1262. ctx = InitPendingUsagesInContext(ctx)
  1263. }
  1264. dryRun := jsonutils.QueryBoolean(data, "dry_run", false)
  1265. // inject tag filters imposed by policy
  1266. data.(*jsonutils.JSONDict).Update(policyResult.Json())
  1267. // 资源实际创建函数
  1268. model, err := DoCreate(manager, ctx, userCred, query, data, ownerId)
  1269. if err != nil {
  1270. // validate failed, clean pending usage
  1271. if CancelPendingUsagesInContext != nil {
  1272. e := CancelPendingUsagesInContext(ctx, userCred)
  1273. if e != nil {
  1274. err = errors.Wrapf(err, "CancelPendingUsagesInContext fail %s", e.Error())
  1275. }
  1276. }
  1277. failErr := manager.OnCreateFailed(ctx, userCred, ownerId, query, data)
  1278. if failErr != nil {
  1279. err = errors.Wrapf(err, "%s", failErr.Error())
  1280. }
  1281. return nil, httperrors.NewGeneralError(err)
  1282. }
  1283. // 伪创建
  1284. if dryRun {
  1285. // dry run, clean pending usage
  1286. if CancelPendingUsagesInContext != nil {
  1287. err := CancelPendingUsagesInContext(ctx, userCred)
  1288. if err != nil {
  1289. return nil, errors.Wrap(err, "CancelPendingUsagesInContext")
  1290. }
  1291. }
  1292. return getItemDetails(manager, model, ctx, userCred, query)
  1293. }
  1294. // 资源创建完成后所需执行的任务(创建完成指在数据库中存在数据)
  1295. func() {
  1296. lockman.LockObject(ctx, model)
  1297. defer lockman.ReleaseObject(ctx, model)
  1298. model.PostCreate(ctx, userCred, ownerId, query, data)
  1299. if err := manager.GetExtraHook().AfterPostCreate(ctx, userCred, ownerId, model, query, data); err != nil {
  1300. logclient.AddActionLogWithContext(ctx, model, logclient.ACT_POST_CREATE_HOOK, err, userCred, false)
  1301. }
  1302. }()
  1303. // 添加操作日志与消息通知
  1304. {
  1305. notes := model.GetShortDesc(ctx)
  1306. OpsLog.LogEvent(model, ACT_CREATE, notes, userCred)
  1307. logclient.AddActionLogWithContext(ctx, model, logclient.ACT_CREATE, notes, userCred, true)
  1308. }
  1309. manager.OnCreateComplete(ctx, []IModel{model}, userCred, ownerId, query, []jsonutils.JSONObject{data})
  1310. return getItemDetails(manager, model, ctx, userCred, query)
  1311. }
  1312. func expandMultiCreateParams(manager IModelManager,
  1313. ctx context.Context,
  1314. userCred mcclient.TokenCredential,
  1315. ownerId mcclient.IIdentityProvider,
  1316. query jsonutils.JSONObject,
  1317. data jsonutils.JSONObject,
  1318. count int,
  1319. ) ([]jsonutils.JSONObject, error) {
  1320. jsonDict, ok := data.(*jsonutils.JSONDict)
  1321. if !ok {
  1322. return nil, httperrors.NewInputParameterError("body is not a json?")
  1323. }
  1324. if manager.HasName() {
  1325. name, _ := jsonDict.GetString("generate_name")
  1326. if len(name) == 0 {
  1327. name, _ = jsonDict.GetString("name")
  1328. if len(name) == 0 {
  1329. return nil, httperrors.NewInputParameterError("Missing name or generate_name")
  1330. }
  1331. jsonDict.Add(jsonutils.NewString(name), "generate_name")
  1332. jsonDict.RemoveIgnoreCase("name")
  1333. }
  1334. }
  1335. ret := make([]jsonutils.JSONObject, count)
  1336. for i := 0; i < count; i += 1 {
  1337. input, err := ExpandBatchCreateData(manager, ctx, userCred, ownerId, query, jsonDict.Copy(), i)
  1338. if err != nil {
  1339. if errors.Cause(err) == MethodNotFoundError {
  1340. ret[i] = jsonDict.Copy()
  1341. } else {
  1342. return nil, errors.Wrap(err, "ExpandBatchCreateData")
  1343. }
  1344. } else {
  1345. ret[i] = input
  1346. }
  1347. }
  1348. return ret, nil
  1349. }
  1350. func (dispatcher *DBModelDispatcher) BatchCreate(ctx context.Context, query jsonutils.JSONObject, data jsonutils.JSONObject, count int, ctxIds []dispatcher.SResourceContext) ([]printutils.SubmitResult, error) {
  1351. userCred := fetchUserCredential(ctx)
  1352. manager := dispatcher.manager.GetMutableInstance(ctx, userCred, query, data)
  1353. ownerId, err := fetchOwnerId(ctx, manager, userCred, data)
  1354. if err != nil {
  1355. return nil, httperrors.NewGeneralError(err)
  1356. }
  1357. if len(ctxIds) > 0 {
  1358. dataDict, ok := data.(*jsonutils.JSONDict)
  1359. if !ok {
  1360. return nil, fmt.Errorf("fail to parse body")
  1361. }
  1362. data, err = fetchContextObjectsIds(manager, ctx, userCred, ctxIds, dataDict)
  1363. if err != nil {
  1364. return nil, err
  1365. }
  1366. }
  1367. var policyResult rbacutils.SPolicyResult
  1368. policyResult, err = isClassRbacAllowed(ctx, manager, userCred, ownerId, policy.PolicyActionCreate)
  1369. if err != nil {
  1370. return nil, errors.Wrap(err, "isClassRbacAllowd")
  1371. }
  1372. data.(*jsonutils.JSONDict).Update(policyResult.Json())
  1373. type sCreateResult struct {
  1374. model IModel
  1375. err error
  1376. }
  1377. var (
  1378. multiData []jsonutils.JSONObject
  1379. // onBatchCreateFail func()
  1380. // validateError error
  1381. )
  1382. if InitPendingUsagesInContext != nil {
  1383. ctx = InitPendingUsagesInContext(ctx)
  1384. }
  1385. createResults, err := func() ([]sCreateResult, error) {
  1386. lockman.LockClass(ctx, manager, GetLockClassKey(manager, ownerId))
  1387. defer lockman.ReleaseClass(ctx, manager, GetLockClassKey(manager, ownerId))
  1388. // invoke only Once
  1389. err = manager.BatchPreValidate(ctx, userCred, ownerId, query, data.(*jsonutils.JSONDict), count)
  1390. if err != nil {
  1391. return nil, errors.Wrap(err, "manager.BatchPreValidate")
  1392. }
  1393. multiData, err = expandMultiCreateParams(manager, ctx, userCred, ownerId, query, data, count)
  1394. if err != nil {
  1395. return nil, errors.Wrap(err, "expandMultiCreateParams")
  1396. }
  1397. // one fail, then all fail
  1398. ret := make([]sCreateResult, len(multiData))
  1399. for i := range multiData {
  1400. var model IModel
  1401. log.Debugf("batchCreateDoCreateItem %d %s", i, multiData[i].String())
  1402. model, err = batchCreateDoCreateItem(manager, ctx, userCred, ownerId, query, multiData[i], i+1)
  1403. if err == nil {
  1404. ret[i] = sCreateResult{model: model, err: nil}
  1405. } else {
  1406. break
  1407. }
  1408. }
  1409. if err != nil {
  1410. for i := range ret {
  1411. if ret[i].model != nil {
  1412. DeleteModel(ctx, userCred, ret[i].model)
  1413. ret[i].model = nil
  1414. }
  1415. ret[i].err = err
  1416. }
  1417. return nil, errors.Wrap(err, "batchCreateDoCreateItem")
  1418. } else {
  1419. return ret, nil
  1420. }
  1421. }()
  1422. if err != nil {
  1423. failErr := manager.OnCreateFailed(ctx, userCred, ownerId, query, data)
  1424. if failErr != nil {
  1425. err = errors.Wrapf(err, "%s", failErr.Error())
  1426. }
  1427. return nil, httperrors.NewGeneralError(errors.Wrap(err, "createResults"))
  1428. }
  1429. results := make([]printutils.SubmitResult, count)
  1430. models := make([]IModel, 0)
  1431. for i := range createResults {
  1432. res := createResults[i]
  1433. result := printutils.SubmitResult{}
  1434. if res.err != nil {
  1435. jsonErr := httperrors.NewGeneralError(res.err)
  1436. result.Status = jsonErr.Code
  1437. result.Data = jsonutils.Marshal(jsonErr)
  1438. } else {
  1439. func() {
  1440. lockman.LockObject(ctx, res.model)
  1441. defer lockman.ReleaseObject(ctx, res.model)
  1442. res.model.PostCreate(ctx, userCred, ownerId, query, data)
  1443. }()
  1444. models = append(models, res.model)
  1445. body, err := getItemDetails(manager, res.model, ctx, userCred, query)
  1446. if err != nil {
  1447. result.Status = 500
  1448. result.Data = jsonutils.NewString(err.Error()) // no translation here
  1449. } else {
  1450. result.Status = 200
  1451. result.Data = body
  1452. }
  1453. }
  1454. results[i] = result
  1455. }
  1456. if len(models) > 0 {
  1457. func() {
  1458. lockman.LockClass(ctx, manager, GetLockClassKey(manager, ownerId))
  1459. defer lockman.ReleaseClass(ctx, manager, GetLockClassKey(manager, ownerId))
  1460. manager.OnCreateComplete(ctx, models, userCred, ownerId, query, multiData)
  1461. }()
  1462. }
  1463. return results, nil
  1464. }
  1465. func (dispatcher *DBModelDispatcher) PerformClassAction(ctx context.Context, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
  1466. // 伪创建,校验创建参数
  1467. if action == "check-create-data" {
  1468. dataDict := data.(*jsonutils.JSONDict)
  1469. dataDict.Set("dry_run", jsonutils.JSONTrue)
  1470. return dispatcher.Create(ctx, query, dataDict, nil)
  1471. }
  1472. // 获取用户信息
  1473. userCred := fetchUserCredential(ctx)
  1474. manager := dispatcher.manager.GetMutableInstance(ctx, userCred, query, data)
  1475. ownerId, err := fetchOwnerId(ctx, manager, userCred, data)
  1476. if err != nil {
  1477. return nil, httperrors.NewGeneralError(err)
  1478. }
  1479. // 锁住一类
  1480. lockman.LockClass(ctx, manager, GetLockClassKey(manager, ownerId))
  1481. defer lockman.ReleaseClass(ctx, manager, GetLockClassKey(manager, ownerId))
  1482. managerValue := reflect.ValueOf(manager)
  1483. return objectPerformAction(manager, nil, managerValue, ctx, userCred, action, query, data)
  1484. }
  1485. func (dispatcher *DBModelDispatcher) PerformAction(ctx context.Context, idStr string, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
  1486. userCred := fetchUserCredential(ctx)
  1487. manager := dispatcher.manager.GetMutableInstance(ctx, userCred, query, data)
  1488. model, err := fetchItem(manager, ctx, userCred, idStr, nil)
  1489. if err == sql.ErrNoRows {
  1490. return nil, httperrors.NewResourceNotFoundError2(manager.Keyword(), idStr)
  1491. } else if err != nil {
  1492. return nil, httperrors.NewGeneralError(err)
  1493. }
  1494. ret, err := func() (jsonutils.JSONObject, error) {
  1495. lockman.LockObject(ctx, model)
  1496. defer lockman.ReleaseObject(ctx, model)
  1497. if err := model.PreCheckPerformAction(ctx, userCred, action, query, data); err != nil {
  1498. return nil, errors.Wrap(err, "PreCheckPerformAction")
  1499. }
  1500. // 通过action与实例执行请求
  1501. return objectPerformAction(manager, model, reflect.ValueOf(model), ctx, userCred, action, query, data)
  1502. }()
  1503. if err != nil {
  1504. logclient.AddActionLogWithContext(ctx, model, action, err, userCred, false)
  1505. }
  1506. return ret, err
  1507. }
  1508. func objectPerformAction(manager IModelManager, model IModel, modelValue reflect.Value, ctx context.Context, userCred mcclient.TokenCredential, action string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
  1509. return reflectDispatcher(manager, model, modelValue, ctx, userCred, policy.PolicyActionPerform, "PerformAction", "Perform", action, query, data)
  1510. }
  1511. func reflectDispatcher(
  1512. // dispatcher *DBModelDispatcher,
  1513. manager IModelManager,
  1514. model IModel,
  1515. modelValue reflect.Value,
  1516. ctx context.Context,
  1517. userCred mcclient.TokenCredential,
  1518. operator string,
  1519. generalFuncName string,
  1520. funcPrefix string,
  1521. spec string,
  1522. query jsonutils.JSONObject,
  1523. data jsonutils.JSONObject,
  1524. ) (jsonutils.JSONObject, error) {
  1525. result, err := reflectDispatcherInternal(
  1526. manager, model, modelValue, ctx, userCred, operator, generalFuncName, funcPrefix, spec, query, data)
  1527. if model != nil && err == nil && result == nil {
  1528. return getItemDetails(manager, model, ctx, userCred, query)
  1529. } else {
  1530. return result, err
  1531. }
  1532. }
  1533. func reflectDispatcherInternal(
  1534. // dispatcher *DBModelDispatcher,
  1535. manager IModelManager,
  1536. model IModel,
  1537. modelValue reflect.Value,
  1538. ctx context.Context,
  1539. userCred mcclient.TokenCredential,
  1540. operator string,
  1541. generalFuncName string,
  1542. funcPrefix string,
  1543. spec string,
  1544. query jsonutils.JSONObject,
  1545. data jsonutils.JSONObject,
  1546. ) (jsonutils.JSONObject, error) {
  1547. isGeneral := false
  1548. // 优先通过action查找该model下的PerformXXX方法
  1549. funcName := fmt.Sprintf("%s%s", funcPrefix, utils.Kebab2Camel(spec, "-"))
  1550. funcValue := modelValue.MethodByName(funcName)
  1551. // 若不存在该方法则根据generalFuncName查找model下的PerformAction方法
  1552. if !funcValue.IsValid() || funcValue.IsNil() {
  1553. funcValue = modelValue.MethodByName(generalFuncName)
  1554. if !funcValue.IsValid() || funcValue.IsNil() {
  1555. return nil, httperrors.NewActionNotFoundError("%s %s %s not found, please check service version, current version: %s",
  1556. manager.Keyword(), operator, spec, version.GetShortString())
  1557. } else {
  1558. isGeneral = true
  1559. funcName = generalFuncName
  1560. }
  1561. }
  1562. var params []interface{}
  1563. if isGeneral {
  1564. params = []interface{}{ctx, userCred, spec, query, data}
  1565. } else {
  1566. params = []interface{}{ctx, userCred, query, data}
  1567. }
  1568. // 若perform指定一类资源,则当前用户对一类资源的权限,否则校验用户对该资源的权限
  1569. var result rbacutils.SPolicyResult
  1570. if model == nil {
  1571. ownerId, err := fetchOwnerId(ctx, manager, userCred, data)
  1572. if err != nil {
  1573. return nil, httperrors.NewGeneralError(err)
  1574. }
  1575. _, err = isClassRbacAllowed(ctx, manager, userCred, ownerId, operator, spec)
  1576. if err != nil {
  1577. return nil, err
  1578. }
  1579. } else {
  1580. var err error
  1581. result, err = isObjectRbacAllowedResult(ctx, model, userCred, operator, spec)
  1582. if err != nil {
  1583. return nil, err
  1584. }
  1585. }
  1586. // 调用反射的方法
  1587. outs, err := callFunc(funcValue, funcName, params...)
  1588. if err != nil {
  1589. return nil, err
  1590. }
  1591. // perform方法返回值为jsonutils.JSONObject,error
  1592. // 对于perform方法返回值数量不为2时,默认不合法
  1593. if len(outs) != 2 {
  1594. return nil, httperrors.NewInternalServerError("Invald %s return value", funcName)
  1595. }
  1596. resVal := outs[0]
  1597. errVal := outs[1].Interface()
  1598. if !gotypes.IsNil(errVal) {
  1599. return nil, errVal.(error)
  1600. } else {
  1601. if model != nil {
  1602. if _, ok := model.(IStandaloneModel); ok {
  1603. Metadata.rawSetValues(ctx, model.Keyword(), model.GetId(), tagutils.TagsetMap2MapString(result.ObjectTags.Flattern()), false, "")
  1604. if model.Keyword() == "project" {
  1605. Metadata.rawSetValues(ctx, model.Keyword(), model.GetId(), tagutils.TagsetMap2MapString(result.ProjectTags.Flattern()), false, "")
  1606. } else if model.Keyword() == "domain" {
  1607. Metadata.rawSetValues(ctx, model.Keyword(), model.GetId(), tagutils.TagsetMap2MapString(result.DomainTags.Flattern()), false, "")
  1608. }
  1609. }
  1610. }
  1611. if gotypes.IsNil(resVal.Interface()) {
  1612. return nil, nil
  1613. } else {
  1614. return ValueToJSONObject(resVal), nil
  1615. }
  1616. }
  1617. }
  1618. func DoUpdate(manager IModelManager, item IModel, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
  1619. lockman.LockObject(ctx, item)
  1620. defer lockman.ReleaseObject(ctx, item)
  1621. return updateItem(manager, item, ctx, userCred, query, data)
  1622. }
  1623. func updateItem(manager IModelManager, item IModel, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
  1624. var err error
  1625. // 校验update入参钩子
  1626. err = item.ValidateUpdateCondition(ctx)
  1627. if err != nil {
  1628. return nil, httperrors.NewGeneralError(errors.Wrapf(err, "ValidateUpdateCondition"))
  1629. }
  1630. dataDict, ok := data.(*jsonutils.JSONDict)
  1631. if !ok {
  1632. return nil, httperrors.NewInternalServerError("Invalid data JSONObject")
  1633. }
  1634. dataDict, err = ValidateUpdateData(item, ctx, userCred, query, dataDict)
  1635. if err != nil {
  1636. return nil, httperrors.NewGeneralError(errors.Wrapf(err, "ValidateUpdateData"))
  1637. }
  1638. if manager.HasName() && dataDict.Contains("name") {
  1639. // validate altername
  1640. nameStr, _ := dataDict.GetString("name")
  1641. err := alterNameValidator(ctx, item, nameStr)
  1642. if err != nil {
  1643. return nil, errors.Wrap(err, "alterNameValidator")
  1644. }
  1645. }
  1646. item.PreUpdate(ctx, userCred, query, dataDict)
  1647. diff, err := Update(item, func() error {
  1648. filterData := dataDict.CopyIncludes(updateFields(manager, userCred)...)
  1649. err = filterData.Unmarshal(item)
  1650. if err != nil {
  1651. return httperrors.NewGeneralError(errors.Wrapf(err, "filterData.Unmarshal"))
  1652. }
  1653. return nil
  1654. })
  1655. if err != nil {
  1656. return nil, httperrors.NewGeneralError(errors.Wrapf(err, "Update"))
  1657. }
  1658. for _, skip := range skipLogFields(manager) {
  1659. delete(diff, skip)
  1660. }
  1661. if len(diff) > 0 {
  1662. OpsLog.LogEvent(item, ACT_UPDATE, diff, userCred)
  1663. logclient.AddActionLogWithContext(ctx, item, logclient.ACT_UPDATE, diff, userCred, true)
  1664. CallUpdateNotifyHook(ctx, userCred, item)
  1665. }
  1666. item.PostUpdate(ctx, userCred, query, data)
  1667. if err := manager.GetExtraHook().AfterPostUpdate(ctx, userCred, item, query, data); err != nil {
  1668. logclient.AddActionLogWithContext(ctx, item, logclient.ACT_POST_UPDATE_HOOK, err, userCred, false)
  1669. }
  1670. return getItemDetails(manager, item, ctx, userCred, query)
  1671. }
  1672. func (dispatcher *DBModelDispatcher) FetchUpdateHeaderData(ctx context.Context, header http.Header) (jsonutils.JSONObject, error) {
  1673. return dispatcher.manager.FetchUpdateHeaderData(ctx, header)
  1674. }
  1675. func (dispatcher *DBModelDispatcher) Update(ctx context.Context, idStr string, query jsonutils.JSONObject, data jsonutils.JSONObject, ctxIds []dispatcher.SResourceContext) (jsonutils.JSONObject, error) {
  1676. userCred := fetchUserCredential(ctx)
  1677. if data == nil {
  1678. data = jsonutils.NewDict()
  1679. }
  1680. manager := dispatcher.manager.GetMutableInstance(ctx, userCred, query, data)
  1681. model, err := fetchItem(manager, ctx, userCred, idStr, nil)
  1682. if err == sql.ErrNoRows {
  1683. return nil, httperrors.NewResourceNotFoundError2(manager.Keyword(), idStr)
  1684. } else if err != nil {
  1685. return nil, httperrors.NewGeneralError(err)
  1686. }
  1687. result, err := isObjectRbacAllowedResult(ctx, model, userCred, policy.PolicyActionUpdate)
  1688. if err != nil {
  1689. return nil, err
  1690. }
  1691. data.(*jsonutils.JSONDict).Update(result.Json())
  1692. if len(ctxIds) > 0 {
  1693. ctxObjs, err := fetchContextObjects(manager, ctx, userCred, ctxIds)
  1694. if err != nil {
  1695. return nil, httperrors.NewGeneralError(err)
  1696. }
  1697. return model.UpdateInContext(ctx, userCred, ctxObjs, query, data)
  1698. } else {
  1699. lockman.LockObject(ctx, model)
  1700. defer lockman.ReleaseObject(ctx, model)
  1701. return updateItem(manager, model, ctx, userCred, query, data)
  1702. }
  1703. }
  1704. func (dispatcher *DBModelDispatcher) UpdateSpec(ctx context.Context, idStr string, spec string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
  1705. userCred := fetchUserCredential(ctx)
  1706. manager := dispatcher.manager.GetMutableInstance(ctx, userCred, query, data)
  1707. model, err := fetchItem(manager, ctx, userCred, idStr, nil)
  1708. if err == sql.ErrNoRows {
  1709. return nil, httperrors.NewResourceNotFoundError2(manager.Keyword(), idStr)
  1710. } else if err != nil {
  1711. return nil, httperrors.NewGeneralError(err)
  1712. }
  1713. lockman.LockObject(ctx, model)
  1714. defer lockman.ReleaseObject(ctx, model)
  1715. return objectUpdateSpec(manager, model, reflect.ValueOf(model), ctx, userCred, spec, query, data)
  1716. }
  1717. func objectUpdateSpec(manager IModelManager, model IModel, modelValue reflect.Value, ctx context.Context, userCred mcclient.TokenCredential, spec string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
  1718. return reflectDispatcher(manager, model, modelValue, ctx, userCred, policy.PolicyActionUpdate, "UpdateSpec", "Update", spec, query, data)
  1719. }
  1720. func DeleteModel(ctx context.Context, userCred mcclient.TokenCredential, item IModel) error {
  1721. // cleanModelUsages(ctx, userCred, item)
  1722. _, err := Update(item, func() error {
  1723. return item.MarkDelete()
  1724. })
  1725. if err != nil {
  1726. return httperrors.NewGeneralError(errors.Wrapf(err, "db.Update"))
  1727. }
  1728. if userCred != nil {
  1729. OpsLog.LogEvent(item, ACT_DELETE, item.GetShortDesc(ctx), userCred)
  1730. logclient.AddSimpleActionLog(item, logclient.ACT_DELETE, item.GetShortDesc(ctx), userCred, true)
  1731. }
  1732. if _, ok := item.(IStandaloneModel); ok && len(item.GetId()) > 0 {
  1733. err := Metadata.RemoveAll(ctx, item, userCred)
  1734. if err != nil {
  1735. return errors.Wrapf(err, "Metadata.RemoveAll")
  1736. }
  1737. }
  1738. return nil
  1739. }
  1740. func RealDeleteModel(ctx context.Context, userCred mcclient.TokenCredential, item IModel) error {
  1741. if len(item.GetId()) == 0 {
  1742. return DeleteModel(ctx, userCred, item)
  1743. }
  1744. _, err := sqlchemy.GetDB().Exec(
  1745. fmt.Sprintf(
  1746. "delete from %s where id = ?",
  1747. item.GetModelManager().TableSpec().Name(),
  1748. ), item.GetId(),
  1749. )
  1750. if err != nil {
  1751. return httperrors.NewGeneralError(errors.Wrapf(err, "db.Update"))
  1752. }
  1753. if userCred != nil {
  1754. OpsLog.LogEvent(item, ACT_DELETE, item.GetShortDesc(ctx), userCred)
  1755. logclient.AddSimpleActionLog(item, logclient.ACT_DELETE, item.GetShortDesc(ctx), userCred, true)
  1756. }
  1757. if _, ok := item.(IStandaloneModel); ok && len(item.GetId()) > 0 {
  1758. err := Metadata.RemoveAll(ctx, item, userCred)
  1759. if err != nil {
  1760. return errors.Wrapf(err, "Metadata.RemoveAll")
  1761. }
  1762. }
  1763. return nil
  1764. }
  1765. func deleteItem(manager IModelManager, model IModel, ctx context.Context, userCred mcclient.TokenCredential, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
  1766. // 获取实例详情
  1767. details, err := getItemDetails(manager, model, ctx, userCred, query)
  1768. if err != nil {
  1769. return nil, httperrors.NewGeneralError(errors.Wrapf(err, "getItemDetails"))
  1770. }
  1771. // 删除校验
  1772. err = ValidateDeleteCondition(model, ctx, details)
  1773. if err != nil {
  1774. return nil, err
  1775. }
  1776. // 删除前钩子
  1777. err = CustomizeDelete(model, ctx, userCred, query, data)
  1778. if err != nil {
  1779. return nil, httperrors.NewGeneralError(errors.Wrapf(err, "CustomizeDelete"))
  1780. }
  1781. model.PreDelete(ctx, userCred)
  1782. // 实际删除
  1783. err = model.Delete(ctx, userCred)
  1784. if err != nil {
  1785. return nil, errors.Wrapf(err, "Delete")
  1786. }
  1787. // 删除后钩子
  1788. model.PostDelete(ctx, userCred)
  1789. if err := model.GetModelManager().GetExtraHook().AfterPostDelete(ctx, userCred, model, query); err != nil {
  1790. logclient.AddActionLogWithContext(ctx, model, logclient.ACT_POST_DELETE_HOOK, err, userCred, false)
  1791. }
  1792. // 避免设置删除状态没有正常返回
  1793. jsonutils.Update(details, model)
  1794. return details, nil
  1795. }
  1796. func (dispatcher *DBModelDispatcher) Delete(ctx context.Context, idstr string, query jsonutils.JSONObject, data jsonutils.JSONObject, ctxIds []dispatcher.SResourceContext) (jsonutils.JSONObject, error) {
  1797. userCred := fetchUserCredential(ctx)
  1798. manager := dispatcher.manager.GetMutableInstance(ctx, userCred, query, data)
  1799. // 找到实例
  1800. model, err := fetchItem(manager, ctx, userCred, idstr, nil)
  1801. if err == sql.ErrNoRows {
  1802. return nil, httperrors.NewResourceNotFoundError2(manager.Keyword(), idstr)
  1803. } else if err != nil {
  1804. return nil, httperrors.NewGeneralError(err)
  1805. }
  1806. // 校验角色
  1807. err = isObjectRbacAllowed(ctx, model, userCred, policy.PolicyActionDelete)
  1808. if err != nil {
  1809. return nil, err
  1810. }
  1811. if len(ctxIds) > 0 {
  1812. ctxObjs, err := fetchContextObjects(manager, ctx, userCred, ctxIds)
  1813. if err != nil {
  1814. return nil, httperrors.NewGeneralError(err)
  1815. }
  1816. return model.DeleteInContext(ctx, userCred, ctxObjs, query, data)
  1817. } else {
  1818. lockman.LockObject(ctx, model)
  1819. defer lockman.ReleaseObject(ctx, model)
  1820. return deleteItem(manager, model, ctx, userCred, query, data)
  1821. }
  1822. }
  1823. func (dispatcher *DBModelDispatcher) DeleteSpec(ctx context.Context, idstr string, spec string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
  1824. userCred := fetchUserCredential(ctx)
  1825. manager := dispatcher.manager.GetMutableInstance(ctx, userCred, query, data)
  1826. model, err := fetchItem(manager, ctx, userCred, idstr, nil)
  1827. if err == sql.ErrNoRows {
  1828. return nil, httperrors.NewResourceNotFoundError2(manager.Keyword(), idstr)
  1829. } else if err != nil {
  1830. return nil, httperrors.NewGeneralError(err)
  1831. }
  1832. lockman.LockObject(ctx, model)
  1833. defer lockman.ReleaseObject(ctx, model)
  1834. return objectDeleteSpec(manager, model, reflect.ValueOf(model), ctx, userCred, spec, query, data)
  1835. }
  1836. func objectDeleteSpec(manager IModelManager, model IModel, modelValue reflect.Value, ctx context.Context, userCred mcclient.TokenCredential, spec string, query jsonutils.JSONObject, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
  1837. return reflectDispatcher(manager, model, modelValue, ctx, userCred, policy.PolicyActionDelete, "DeleteSpec", "Delete", spec, query, data)
  1838. }