// Copyright 2019 Yunion // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package misc import ( "fmt" "time" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/cloudmon/options" "yunion.io/x/onecloud/pkg/util/influxdb" ) type SHttpStats struct { HttpCode2xx float64 `json:"duration.2XX"` HttpCode4xx float64 `json:"duration.4XX"` HttpCode5xx float64 `json:"duration.5XX"` HitHttpCode2xx int64 `json:"hit.2XX"` HitHttpCode4xx int64 `json:"hit.4XX"` HitHttpCode5xx int64 `json:"hit.5XX"` Method string `json:"method"` Name string `json:"name"` Path string `json:"path"` } type sApiHttpStats struct { SHttpStats Paths []SHttpStats `json:"paths"` } func (apiStats *sApiHttpStats) convertSnapshot(now time.Time) *sHttpStatsSnapshot { snapshot := &sHttpStatsSnapshot{ snapshotAt: now, stats: map[string]*SHttpStats{}, } apiStats.SHttpStats.Method = "any" apiStats.SHttpStats.Path = "any" apiStats.SHttpStats.Name = "any" snapshot.stats[getStatsKey(apiStats.SHttpStats.Method, apiStats.SHttpStats.Path, apiStats.SHttpStats.Name)] = &apiStats.SHttpStats for i := range apiStats.Paths { pathStats := apiStats.Paths[i] snapshot.stats[getStatsKey(pathStats.Method, pathStats.Path, pathStats.Name)] = &pathStats } return snapshot } type sHttpStatsExt struct { Duration2xx float64 `json:"duration_2xx"` Duration4xx float64 `json:"duration_4xx"` Duration5xx float64 `json:"duration_5xx"` Hit2xx int64 `json:"hit_2xx"` Hit4xx int64 `json:"hit_4xx"` Hit5xx int64 `json:"hit_5xx"` Duration2xxDiff float64 `json:"duration_2xx_diff"` Duration4xxDiff float64 `json:"duration_4xx_diff"` Duration5xxDiff float64 `json:"duration_5xx_diff"` Hit2xxDiff int64 `json:"hit_2xx_diff"` Hit4xxDiff int64 `json:"hit_4xx_diff"` Hit5xxDiff int64 `json:"hit_5xx_diff"` Method string `json:"method"` Path string `json:"path"` Name string `json:"name"` HasDiff bool `json:"has_diff"` } func (v sHttpStatsExt) DelayMs2xx() float64 { if v.Hit2xxDiff > 0 { return v.Duration2xxDiff / float64(v.Hit2xxDiff) } return -1 } func (v sHttpStatsExt) Qps2xx(interval time.Duration) float64 { if interval > 0 { return float64(v.Hit2xxDiff) / interval.Seconds() } return -1 } func (v sHttpStatsExt) DelayMs4xx() float64 { if v.Hit4xxDiff > 0 { return v.Duration4xxDiff / float64(v.Hit4xxDiff) } return -1 } func (v sHttpStatsExt) Qps4xx(interval time.Duration) float64 { if interval > 0 { return float64(v.Hit4xxDiff) / interval.Seconds() } return -1 } func (v sHttpStatsExt) DelayMs5xx() float64 { if v.Hit5xxDiff > 0 { return v.Duration5xxDiff / float64(v.Hit5xxDiff) } return -1 } func (v sHttpStatsExt) Qps5xx(interval time.Duration) float64 { if interval > 0 { return float64(v.Hit5xxDiff) / interval.Seconds() } return -1 } func (v sHttpStatsExt) Qps(interval time.Duration) float64 { if interval > 0 { return float64(v.Hit2xxDiff+v.Hit4xxDiff+v.Hit5xxDiff) / interval.Seconds() } return -1 } func (v sHttpStatsExt) DelayMs() float64 { if v.HitDiff() > 0 { return v.DurationMsDiff() / float64(v.HitDiff()) } return -1 } func (v sHttpStatsExt) DurationMs() float64 { return v.Duration2xx + v.Duration4xx + v.Duration5xx } func (v sHttpStatsExt) Hit() int64 { return v.Hit2xx + v.Hit4xx + v.Hit5xx } func (v sHttpStatsExt) DurationMsDiff() float64 { return v.Duration2xxDiff + v.Duration4xxDiff + v.Duration5xxDiff } func (v sHttpStatsExt) HitDiff() int64 { return v.Hit2xxDiff + v.Hit4xxDiff + v.Hit5xxDiff } func (v sHttpStatsExt) Percent2xx() float64 { if v.DurationMsDiff() > 0 { return v.Duration2xxDiff * 100 / v.DurationMsDiff() } return -1 } func (v sHttpStatsExt) PercentHit2xx() float64 { if v.HitDiff() > 0 { return float64(v.Hit2xxDiff) * 100 / float64(v.HitDiff()) } return -1 } func (v sHttpStatsExt) Percent4xx() float64 { if v.DurationMsDiff() > 0 { return v.Duration4xxDiff * 100 / v.DurationMsDiff() } return -1 } func (v sHttpStatsExt) PercentHit4xx() float64 { if v.HitDiff() > 0 { return float64(v.Hit4xxDiff) * 100 / float64(v.HitDiff()) } return -1 } func (v sHttpStatsExt) Percent5xx() float64 { if v.DurationMsDiff() > 0 { return v.Duration5xxDiff * 100 / v.DurationMsDiff() } return -1 } func (v sHttpStatsExt) PercentHit5xx() float64 { if v.HitDiff() > 0 { return float64(v.Hit5xxDiff) * 100 / float64(v.HitDiff()) } return -1 } type sHttpStatsSnapshot struct { snapshotAt time.Time stats map[string]*SHttpStats } type sHttpStatsDiff struct { snapshotAt time.Time interval time.Duration stats map[string]*sHttpStatsExt } func calculateHttpStatsDiff(prevSnap *sHttpStatsSnapshot, nowSnap *sHttpStatsSnapshot) *sHttpStatsDiff { diff := sHttpStatsDiff{ snapshotAt: nowSnap.snapshotAt, stats: map[string]*sHttpStatsExt{}, } if prevSnap != nil { rootKey := getStatsKey("any", "any", "any") prevRoot := prevSnap.stats[rootKey] nowRoot := nowSnap.stats[rootKey] if prevRoot.HttpCode2xx > nowRoot.HttpCode2xx || prevRoot.HttpCode4xx > nowRoot.HttpCode4xx || prevRoot.HttpCode5xx > nowRoot.HttpCode5xx || prevRoot.HitHttpCode2xx > nowRoot.HitHttpCode2xx || prevRoot.HitHttpCode4xx > nowRoot.HitHttpCode4xx || prevRoot.HitHttpCode5xx > nowRoot.HitHttpCode5xx { // detect a reset, skip this round prevSnap = nil } } if prevSnap != nil { diff.interval = nowSnap.snapshotAt.Sub(prevSnap.snapshotAt) } else { intvMin := options.Options.CollectServiceMetricIntervalMinute if intvMin <= 0 { intvMin = 1 } diff.interval = time.Duration(intvMin) * time.Minute } for k := range nowSnap.stats { v := nowSnap.stats[k] diffStats := sHttpStatsExt{ Duration2xx: v.HttpCode2xx, Duration4xx: v.HttpCode4xx, Duration5xx: v.HttpCode5xx, Hit2xx: v.HitHttpCode2xx, Hit4xx: v.HitHttpCode4xx, Hit5xx: v.HitHttpCode5xx, Method: v.Method, Path: v.Path, Name: v.Name, } if prevSnap != nil { if prevStats, ok := prevSnap.stats[k]; ok { diffStats.HasDiff = true diffStats.Duration2xxDiff = v.HttpCode2xx - prevStats.HttpCode2xx diffStats.Duration4xxDiff = v.HttpCode4xx - prevStats.HttpCode4xx diffStats.Duration5xxDiff = v.HttpCode5xx - prevStats.HttpCode5xx diffStats.Hit2xxDiff = v.HitHttpCode2xx - prevStats.HitHttpCode2xx diffStats.Hit4xxDiff = v.HitHttpCode4xx - prevStats.HitHttpCode4xx diffStats.Hit5xxDiff = v.HitHttpCode5xx - prevStats.HitHttpCode5xx } } diff.stats[k] = &diffStats } return &diff } var ( httpStatsSnapshot = map[string]*sHttpStatsSnapshot{} ) func getStatsKey(method, path, name string) string { return fmt.Sprintf("%s.%s.%s", method, path, name) } func getSnapshotKey(serviceName, url string) string { return fmt.Sprintf("%s.%s", serviceName, url) } func updateHttpStatsSnapshot(serviceName string, url string, now time.Time, apiStats sApiHttpStats, serviceType, regionId, version string) []influxdb.SMetricData { snapshot := apiStats.convertSnapshot(now) snapshotKey := getSnapshotKey(serviceName, url) var vdiffStats *sHttpStatsDiff if prevSnap, ok := httpStatsSnapshot[snapshotKey]; ok { vdiffStats = calculateHttpStatsDiff(prevSnap, snapshot) } else { // no prev records, just add vdiffStats = calculateHttpStatsDiff(nil, snapshot) } httpStatsSnapshot[snapshotKey] = snapshot metrics := vdiffStats.metrics(serviceName, serviceType, regionId, version) log.Debugf("updateHttpStatsSnapshot %s %s snapshotAt: %s diffAt: %s intval: %f metrics: %d", serviceName, url, snapshot.snapshotAt, vdiffStats.snapshotAt, vdiffStats.interval.Seconds(), len(metrics)) return metrics } func appendMetric(metrics []influxdb.SKeyValue, key string, v float64) []influxdb.SKeyValue { if v >= 0 { metrics = append(metrics, influxdb.SKeyValue{ Key: key, Value: fmt.Sprintf("%f", v), }) } return metrics } func (diff *sHttpStatsDiff) metrics(service, serviceType, regionId, version string) []influxdb.SMetricData { metrics := make([]influxdb.SMetricData, 0) genTags := func(v *sHttpStatsExt) []influxdb.SKeyValue { return []influxdb.SKeyValue{ { Key: "service", Value: service, }, { Key: "service_type", Value: serviceType, }, { Key: "region", Value: regionId, }, { Key: "version", Value: version, }, { Key: "method", Value: v.Method, }, { Key: "path", Value: v.Path, }, { Key: "name", Value: v.Name, }, { Key: "interval_secs", Value: fmt.Sprintf("%f", diff.interval.Seconds()), }, } } for k := range diff.stats { v := diff.stats[k] // ignore stats with no hit if v.Hit() <= 0 { continue } // ignore stats with no diff if v.HasDiff && v.HitDiff() <= 0 { continue } metric := influxdb.SMetricData{ Name: METIRCY_TYPE_HTTP_REQUST, Timestamp: diff.snapshotAt, Tags: genTags(v), Metrics: []influxdb.SKeyValue{ { Key: "duration_ms_any", Value: fmt.Sprintf("%f", v.DurationMs()), }, { Key: "hit_any", Value: fmt.Sprintf("%d", v.Hit()), }, }, } if v.HasDiff { metric.Metrics = appendMetric(metric.Metrics, "dura_ms_delta_any", v.DurationMsDiff()) metric.Metrics = appendMetric(metric.Metrics, "hit_delta_any", float64(v.HitDiff())) metric.Metrics = appendMetric(metric.Metrics, "delay_ms_any", v.DelayMs()) metric.Metrics = appendMetric(metric.Metrics, "qps_any", v.Qps(diff.interval)) } metric.Metrics = append(metric.Metrics, []influxdb.SKeyValue{ { Key: "duration_ms_2xx", Value: fmt.Sprintf("%f", v.Duration2xx), }, { Key: "hit_2xx", Value: fmt.Sprintf("%d", v.Hit2xx), }, }...) if v.HasDiff { metric.Metrics = appendMetric(metric.Metrics, "dura_ms_delta_2xx", v.Duration2xxDiff) metric.Metrics = appendMetric(metric.Metrics, "hit_delta_2xx", float64(v.Hit2xxDiff)) metric.Metrics = appendMetric(metric.Metrics, "delay_ms_2xx", v.DelayMs2xx()) metric.Metrics = appendMetric(metric.Metrics, "percent_hit_2xx", v.PercentHit2xx()) metric.Metrics = appendMetric(metric.Metrics, "percent_duration_2xx", v.Percent2xx()) metric.Metrics = appendMetric(metric.Metrics, "qps_2xx", v.Qps2xx(diff.interval)) } metric.Metrics = append(metric.Metrics, []influxdb.SKeyValue{ { Key: "duration_ms_4xx", Value: fmt.Sprintf("%f", v.Duration4xx), }, { Key: "hit_4xx", Value: fmt.Sprintf("%d", v.Hit4xx), }, }...) if v.HasDiff { metric.Metrics = appendMetric(metric.Metrics, "dura_ms_delta_4xx", v.Duration4xxDiff) metric.Metrics = appendMetric(metric.Metrics, "hit_delta_4xx", float64(v.Hit4xxDiff)) metric.Metrics = appendMetric(metric.Metrics, "delay_ms_4xx", v.DelayMs4xx()) metric.Metrics = appendMetric(metric.Metrics, "percent_hit_4xx", v.PercentHit4xx()) metric.Metrics = appendMetric(metric.Metrics, "percent_duration_4xx", v.Percent4xx()) metric.Metrics = appendMetric(metric.Metrics, "qps_4xx", v.Qps4xx(diff.interval)) } metric.Metrics = append(metric.Metrics, []influxdb.SKeyValue{ { Key: "duration_ms_5xx", Value: fmt.Sprintf("%f", v.Duration5xx), }, { Key: "hit_5xx", Value: fmt.Sprintf("%d", v.Hit5xx), }, }...) if v.HasDiff { metric.Metrics = appendMetric(metric.Metrics, "dura_ms_delta_5xx", v.Duration5xxDiff) metric.Metrics = appendMetric(metric.Metrics, "hit_delta_5xx", float64(v.Hit5xxDiff)) metric.Metrics = appendMetric(metric.Metrics, "delay_ms_5xx", v.DelayMs5xx()) metric.Metrics = appendMetric(metric.Metrics, "percent_hit_5xx", v.PercentHit5xx()) metric.Metrics = appendMetric(metric.Metrics, "percent_duration_5xx", v.Percent5xx()) metric.Metrics = appendMetric(metric.Metrics, "qps_5xx", v.Qps5xx(diff.interval)) } metrics = append(metrics, metric) } return metrics }