metrics.go 4.7 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110
  1. // Unless explicitly stated otherwise all files in this repository are licensed
  2. // under the Apache License Version 2.0.
  3. // This product includes software developed at Datadog (https://www.datadoghq.com/).
  4. // Copyright 2016 Datadog, Inc.
  5. package tracer
  6. import (
  7. "runtime"
  8. "runtime/debug"
  9. "sync/atomic"
  10. "time"
  11. "gopkg.in/DataDog/dd-trace-go.v1/internal/log"
  12. )
  13. // defaultMetricsReportInterval specifies the interval at which runtime metrics will
  14. // be reported.
  15. const defaultMetricsReportInterval = 10 * time.Second
  16. type statsdClient interface {
  17. Incr(name string, tags []string, rate float64) error
  18. Count(name string, value int64, tags []string, rate float64) error
  19. Gauge(name string, value float64, tags []string, rate float64) error
  20. Timing(name string, value time.Duration, tags []string, rate float64) error
  21. Flush() error
  22. Close() error
  23. }
  24. // reportRuntimeMetrics periodically reports go runtime metrics at
  25. // the given interval.
  26. func (t *tracer) reportRuntimeMetrics(interval time.Duration) {
  27. var ms runtime.MemStats
  28. gc := debug.GCStats{
  29. // When len(stats.PauseQuantiles) is 5, it will be filled with the
  30. // minimum, 25%, 50%, 75%, and maximum pause times. See the documentation
  31. // for (runtime/debug).ReadGCStats.
  32. PauseQuantiles: make([]time.Duration, 5),
  33. }
  34. tick := time.NewTicker(interval)
  35. defer tick.Stop()
  36. for {
  37. select {
  38. case <-tick.C:
  39. log.Debug("Reporting runtime metrics...")
  40. runtime.ReadMemStats(&ms)
  41. debug.ReadGCStats(&gc)
  42. statsd := t.statsd
  43. // CPU statistics
  44. statsd.Gauge("runtime.go.num_cpu", float64(runtime.NumCPU()), nil, 1)
  45. statsd.Gauge("runtime.go.num_goroutine", float64(runtime.NumGoroutine()), nil, 1)
  46. statsd.Gauge("runtime.go.num_cgo_call", float64(runtime.NumCgoCall()), nil, 1)
  47. // General statistics
  48. statsd.Gauge("runtime.go.mem_stats.alloc", float64(ms.Alloc), nil, 1)
  49. statsd.Gauge("runtime.go.mem_stats.total_alloc", float64(ms.TotalAlloc), nil, 1)
  50. statsd.Gauge("runtime.go.mem_stats.sys", float64(ms.Sys), nil, 1)
  51. statsd.Gauge("runtime.go.mem_stats.lookups", float64(ms.Lookups), nil, 1)
  52. statsd.Gauge("runtime.go.mem_stats.mallocs", float64(ms.Mallocs), nil, 1)
  53. statsd.Gauge("runtime.go.mem_stats.frees", float64(ms.Frees), nil, 1)
  54. // Heap memory statistics
  55. statsd.Gauge("runtime.go.mem_stats.heap_alloc", float64(ms.HeapAlloc), nil, 1)
  56. statsd.Gauge("runtime.go.mem_stats.heap_sys", float64(ms.HeapSys), nil, 1)
  57. statsd.Gauge("runtime.go.mem_stats.heap_idle", float64(ms.HeapIdle), nil, 1)
  58. statsd.Gauge("runtime.go.mem_stats.heap_inuse", float64(ms.HeapInuse), nil, 1)
  59. statsd.Gauge("runtime.go.mem_stats.heap_released", float64(ms.HeapReleased), nil, 1)
  60. statsd.Gauge("runtime.go.mem_stats.heap_objects", float64(ms.HeapObjects), nil, 1)
  61. // Stack memory statistics
  62. statsd.Gauge("runtime.go.mem_stats.stack_inuse", float64(ms.StackInuse), nil, 1)
  63. statsd.Gauge("runtime.go.mem_stats.stack_sys", float64(ms.StackSys), nil, 1)
  64. // Off-heap memory statistics
  65. statsd.Gauge("runtime.go.mem_stats.m_span_inuse", float64(ms.MSpanInuse), nil, 1)
  66. statsd.Gauge("runtime.go.mem_stats.m_span_sys", float64(ms.MSpanSys), nil, 1)
  67. statsd.Gauge("runtime.go.mem_stats.m_cache_inuse", float64(ms.MCacheInuse), nil, 1)
  68. statsd.Gauge("runtime.go.mem_stats.m_cache_sys", float64(ms.MCacheSys), nil, 1)
  69. statsd.Gauge("runtime.go.mem_stats.buck_hash_sys", float64(ms.BuckHashSys), nil, 1)
  70. statsd.Gauge("runtime.go.mem_stats.gc_sys", float64(ms.GCSys), nil, 1)
  71. statsd.Gauge("runtime.go.mem_stats.other_sys", float64(ms.OtherSys), nil, 1)
  72. // Garbage collector statistics
  73. statsd.Gauge("runtime.go.mem_stats.next_gc", float64(ms.NextGC), nil, 1)
  74. statsd.Gauge("runtime.go.mem_stats.last_gc", float64(ms.LastGC), nil, 1)
  75. statsd.Gauge("runtime.go.mem_stats.pause_total_ns", float64(ms.PauseTotalNs), nil, 1)
  76. statsd.Gauge("runtime.go.mem_stats.num_gc", float64(ms.NumGC), nil, 1)
  77. statsd.Gauge("runtime.go.mem_stats.num_forced_gc", float64(ms.NumForcedGC), nil, 1)
  78. statsd.Gauge("runtime.go.mem_stats.gc_cpu_fraction", ms.GCCPUFraction, nil, 1)
  79. for i, p := range []string{"min", "25p", "50p", "75p", "max"} {
  80. statsd.Gauge("runtime.go.gc_stats.pause_quantiles."+p, float64(gc.PauseQuantiles[i]), nil, 1)
  81. }
  82. case <-t.stop:
  83. return
  84. }
  85. }
  86. }
  87. func (t *tracer) reportHealthMetrics(interval time.Duration) {
  88. ticker := time.NewTicker(interval)
  89. defer ticker.Stop()
  90. for {
  91. select {
  92. case <-ticker.C:
  93. t.statsd.Count("datadog.tracer.spans_started", int64(atomic.SwapUint32(&t.spansStarted, 0)), nil, 1)
  94. t.statsd.Count("datadog.tracer.spans_finished", int64(atomic.SwapUint32(&t.spansFinished, 0)), nil, 1)
  95. t.statsd.Count("datadog.tracer.traces_dropped", int64(atomic.SwapUint32(&t.tracesDropped, 0)), []string{"reason:trace_too_large"}, 1)
  96. case <-t.stop:
  97. return
  98. }
  99. }
  100. }