fsHandler.go 3.8 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155
  1. // Copyright 2015 Google Inc. All Rights Reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. // Handler for Docker containers.
  15. package common
  16. import (
  17. "fmt"
  18. "sync"
  19. "time"
  20. "github.com/google/cadvisor/fs"
  21. "k8s.io/klog/v2"
  22. )
  23. type FsHandler interface {
  24. Start()
  25. Usage() FsUsage
  26. Stop()
  27. }
  28. type FsUsage struct {
  29. BaseUsageBytes uint64
  30. TotalUsageBytes uint64
  31. InodeUsage uint64
  32. }
  33. type realFsHandler struct {
  34. sync.RWMutex
  35. lastUpdate time.Time
  36. usage FsUsage
  37. period time.Duration
  38. minPeriod time.Duration
  39. rootfs string
  40. extraDir string
  41. fsInfo fs.FsInfo
  42. // Tells the container to stop.
  43. stopChan chan struct{}
  44. }
  45. const (
  46. maxBackoffFactor = 20
  47. )
  48. const DefaultPeriod = time.Minute
  49. var _ FsHandler = &realFsHandler{}
  50. func NewFsHandler(period time.Duration, rootfs, extraDir string, fsInfo fs.FsInfo) FsHandler {
  51. return &realFsHandler{
  52. lastUpdate: time.Time{},
  53. usage: FsUsage{},
  54. period: period,
  55. minPeriod: period,
  56. rootfs: rootfs,
  57. extraDir: extraDir,
  58. fsInfo: fsInfo,
  59. stopChan: make(chan struct{}, 1),
  60. }
  61. }
  62. func (fh *realFsHandler) update() error {
  63. var (
  64. rootUsage, extraUsage fs.UsageInfo
  65. rootErr, extraErr error
  66. )
  67. // TODO(vishh): Add support for external mounts.
  68. if fh.rootfs != "" {
  69. rootUsage, rootErr = fh.fsInfo.GetDirUsage(fh.rootfs)
  70. }
  71. if fh.extraDir != "" {
  72. extraUsage, extraErr = fh.fsInfo.GetDirUsage(fh.extraDir)
  73. }
  74. // Wait to handle errors until after all operartions are run.
  75. // An error in one will not cause an early return, skipping others
  76. fh.Lock()
  77. defer fh.Unlock()
  78. fh.lastUpdate = time.Now()
  79. if fh.rootfs != "" && rootErr == nil {
  80. fh.usage.InodeUsage = rootUsage.Inodes
  81. fh.usage.BaseUsageBytes = rootUsage.Bytes
  82. fh.usage.TotalUsageBytes = rootUsage.Bytes
  83. }
  84. if fh.extraDir != "" && extraErr == nil {
  85. if fh.rootfs != "" {
  86. fh.usage.TotalUsageBytes += extraUsage.Bytes
  87. } else {
  88. // rootfs is empty, totalUsageBytes use extra usage bytes
  89. fh.usage.TotalUsageBytes = extraUsage.Bytes
  90. }
  91. }
  92. // Combine errors into a single error to return
  93. if rootErr != nil || extraErr != nil {
  94. return fmt.Errorf("rootDiskErr: %v, extraDiskErr: %v", rootErr, extraErr)
  95. }
  96. return nil
  97. }
  98. func (fh *realFsHandler) trackUsage() {
  99. longOp := time.Second
  100. for {
  101. start := time.Now()
  102. if err := fh.update(); err != nil {
  103. klog.Errorf("failed to collect filesystem stats - %v", err)
  104. fh.period = fh.period * 2
  105. if fh.period > maxBackoffFactor*fh.minPeriod {
  106. fh.period = maxBackoffFactor * fh.minPeriod
  107. }
  108. } else {
  109. fh.period = fh.minPeriod
  110. }
  111. duration := time.Since(start)
  112. if duration > longOp {
  113. // adapt longOp time so that message doesn't continue to print
  114. // if the long duration is persistent either because of slow
  115. // disk or lots of containers.
  116. longOp = longOp + time.Second
  117. klog.V(2).Infof("fs: disk usage and inodes count on following dirs took %v: %v; will not log again for this container unless duration exceeds %v", duration, []string{fh.rootfs, fh.extraDir}, longOp)
  118. }
  119. select {
  120. case <-fh.stopChan:
  121. return
  122. case <-time.After(fh.period):
  123. }
  124. }
  125. }
  126. func (fh *realFsHandler) Start() {
  127. go fh.trackUsage()
  128. }
  129. func (fh *realFsHandler) Stop() {
  130. close(fh.stopChan)
  131. }
  132. func (fh *realFsHandler) Usage() FsUsage {
  133. fh.RLock()
  134. defer fh.RUnlock()
  135. return fh.usage
  136. }