fs.go 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811
  1. // Copyright 2014 Google Inc. All Rights Reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. //go:build linux
  15. // +build linux
  16. // Provides Filesystem Stats
  17. package fs
  18. import (
  19. "bufio"
  20. "fmt"
  21. "io/ioutil"
  22. "os"
  23. "os/exec"
  24. "path"
  25. "path/filepath"
  26. "regexp"
  27. "strconv"
  28. "strings"
  29. "syscall"
  30. zfs "github.com/mistifyio/go-zfs"
  31. mount "github.com/moby/sys/mountinfo"
  32. "github.com/google/cadvisor/devicemapper"
  33. "github.com/google/cadvisor/utils"
  34. "k8s.io/klog/v2"
  35. )
  36. const (
  37. LabelSystemRoot = "root"
  38. LabelDockerImages = "docker-images"
  39. LabelCrioImages = "crio-images"
  40. DriverStatusPoolName = "Pool Name"
  41. DriverStatusDataLoopFile = "Data loop file"
  42. )
  43. const (
  44. // The block size in bytes.
  45. statBlockSize uint64 = 512
  46. // The maximum number of `disk usage` tasks that can be running at once.
  47. maxConcurrentOps = 20
  48. )
  49. // A pool for restricting the number of consecutive `du` and `find` tasks running.
  50. var pool = make(chan struct{}, maxConcurrentOps)
  51. func init() {
  52. for i := 0; i < maxConcurrentOps; i++ {
  53. releaseToken()
  54. }
  55. }
  56. func claimToken() {
  57. <-pool
  58. }
  59. func releaseToken() {
  60. pool <- struct{}{}
  61. }
  62. type partition struct {
  63. mountpoint string
  64. major uint
  65. minor uint
  66. fsType string
  67. blockSize uint
  68. }
  69. type RealFsInfo struct {
  70. // Map from block device path to partition information.
  71. partitions map[string]partition
  72. // Map from label to block device path.
  73. // Labels are intent-specific tags that are auto-detected.
  74. labels map[string]string
  75. // Map from mountpoint to mount information.
  76. mounts map[string]mount.Info
  77. // devicemapper client
  78. dmsetup devicemapper.DmsetupClient
  79. // fsUUIDToDeviceName is a map from the filesystem UUID to its device name.
  80. fsUUIDToDeviceName map[string]string
  81. }
  82. func NewFsInfo(context Context) (FsInfo, error) {
  83. fileReader, err := os.Open("/proc/self/mountinfo")
  84. if err != nil {
  85. return nil, err
  86. }
  87. mounts, err := mount.GetMountsFromReader(fileReader, nil)
  88. if err != nil {
  89. return nil, err
  90. }
  91. fsUUIDToDeviceName, err := getFsUUIDToDeviceNameMap()
  92. if err != nil {
  93. // UUID is not always available across different OS distributions.
  94. // Do not fail if there is an error.
  95. klog.Warningf("Failed to get disk UUID mapping, getting disk info by uuid will not work: %v", err)
  96. }
  97. // Avoid devicemapper container mounts - these are tracked by the ThinPoolWatcher
  98. excluded := []string{fmt.Sprintf("%s/devicemapper/mnt", context.Docker.Root)}
  99. fsInfo := &RealFsInfo{
  100. partitions: processMounts(mounts, excluded),
  101. labels: make(map[string]string),
  102. mounts: make(map[string]mount.Info),
  103. dmsetup: devicemapper.NewDmsetupClient(),
  104. fsUUIDToDeviceName: fsUUIDToDeviceName,
  105. }
  106. for _, mnt := range mounts {
  107. fsInfo.mounts[mnt.Mountpoint] = *mnt
  108. }
  109. // need to call this before the log line below printing out the partitions, as this function may
  110. // add a "partition" for devicemapper to fsInfo.partitions
  111. fsInfo.addDockerImagesLabel(context, mounts)
  112. fsInfo.addCrioImagesLabel(context, mounts)
  113. klog.V(1).Infof("Filesystem UUIDs: %+v", fsInfo.fsUUIDToDeviceName)
  114. klog.V(1).Infof("Filesystem partitions: %+v", fsInfo.partitions)
  115. fsInfo.addSystemRootLabel(mounts)
  116. return fsInfo, nil
  117. }
  118. // getFsUUIDToDeviceNameMap creates the filesystem uuid to device name map
  119. // using the information in /dev/disk/by-uuid. If the directory does not exist,
  120. // this function will return an empty map.
  121. func getFsUUIDToDeviceNameMap() (map[string]string, error) {
  122. const dir = "/dev/disk/by-uuid"
  123. if _, err := os.Stat(dir); os.IsNotExist(err) {
  124. return make(map[string]string), nil
  125. }
  126. files, err := ioutil.ReadDir(dir)
  127. if err != nil {
  128. return nil, err
  129. }
  130. fsUUIDToDeviceName := make(map[string]string)
  131. for _, file := range files {
  132. fpath := filepath.Join(dir, file.Name())
  133. target, err := os.Readlink(fpath)
  134. if err != nil {
  135. klog.Warningf("Failed to resolve symlink for %q", fpath)
  136. continue
  137. }
  138. device, err := filepath.Abs(filepath.Join(dir, target))
  139. if err != nil {
  140. return nil, fmt.Errorf("failed to resolve the absolute path of %q", filepath.Join(dir, target))
  141. }
  142. fsUUIDToDeviceName[file.Name()] = device
  143. }
  144. return fsUUIDToDeviceName, nil
  145. }
  146. func processMounts(mounts []*mount.Info, excludedMountpointPrefixes []string) map[string]partition {
  147. partitions := make(map[string]partition)
  148. supportedFsType := map[string]bool{
  149. // all ext and nfs systems are checked through prefix
  150. // because there are a number of families (e.g., ext3, ext4, nfs3, nfs4...)
  151. "btrfs": true,
  152. "overlay": true,
  153. "tmpfs": true,
  154. "xfs": true,
  155. "zfs": true,
  156. }
  157. for _, mnt := range mounts {
  158. if !strings.HasPrefix(mnt.FSType, "ext") && !strings.HasPrefix(mnt.FSType, "nfs") &&
  159. !supportedFsType[mnt.FSType] {
  160. continue
  161. }
  162. // Avoid bind mounts, exclude tmpfs.
  163. if _, ok := partitions[mnt.Source]; ok {
  164. if mnt.FSType != "tmpfs" {
  165. continue
  166. }
  167. }
  168. hasPrefix := false
  169. for _, prefix := range excludedMountpointPrefixes {
  170. if strings.HasPrefix(mnt.Mountpoint, prefix) {
  171. hasPrefix = true
  172. break
  173. }
  174. }
  175. if hasPrefix {
  176. continue
  177. }
  178. // using mountpoint to replace device once fstype it tmpfs
  179. if mnt.FSType == "tmpfs" {
  180. mnt.Source = mnt.Mountpoint
  181. }
  182. // btrfs fix: following workaround fixes wrong btrfs Major and Minor Ids reported in /proc/self/mountinfo.
  183. // instead of using values from /proc/self/mountinfo we use stat to get Ids from btrfs mount point
  184. if mnt.FSType == "btrfs" && mnt.Major == 0 && strings.HasPrefix(mnt.Source, "/dev/") {
  185. major, minor, err := getBtrfsMajorMinorIds(mnt)
  186. if err != nil {
  187. klog.Warningf("%s", err)
  188. } else {
  189. mnt.Major = major
  190. mnt.Minor = minor
  191. }
  192. }
  193. // overlay fix: Making mount source unique for all overlay mounts, using the mount's major and minor ids.
  194. if mnt.FSType == "overlay" {
  195. mnt.Source = fmt.Sprintf("%s_%d-%d", mnt.Source, mnt.Major, mnt.Minor)
  196. }
  197. partitions[mnt.Source] = partition{
  198. fsType: mnt.FSType,
  199. mountpoint: mnt.Mountpoint,
  200. major: uint(mnt.Major),
  201. minor: uint(mnt.Minor),
  202. }
  203. }
  204. return partitions
  205. }
  206. // getDockerDeviceMapperInfo returns information about the devicemapper device and "partition" if
  207. // docker is using devicemapper for its storage driver. If a loopback device is being used, don't
  208. // return any information or error, as we want to report based on the actual partition where the
  209. // loopback file resides, inside of the loopback file itself.
  210. func (i *RealFsInfo) getDockerDeviceMapperInfo(context DockerContext) (string, *partition, error) {
  211. if context.Driver != DeviceMapper.String() {
  212. return "", nil, nil
  213. }
  214. dataLoopFile := context.DriverStatus[DriverStatusDataLoopFile]
  215. if len(dataLoopFile) > 0 {
  216. return "", nil, nil
  217. }
  218. dev, major, minor, blockSize, err := dockerDMDevice(context.DriverStatus, i.dmsetup)
  219. if err != nil {
  220. return "", nil, err
  221. }
  222. return dev, &partition{
  223. fsType: DeviceMapper.String(),
  224. major: major,
  225. minor: minor,
  226. blockSize: blockSize,
  227. }, nil
  228. }
  229. // addSystemRootLabel attempts to determine which device contains the mount for /.
  230. func (i *RealFsInfo) addSystemRootLabel(mounts []*mount.Info) {
  231. for _, m := range mounts {
  232. if m.Mountpoint == "/" {
  233. i.partitions[m.Source] = partition{
  234. fsType: m.FSType,
  235. mountpoint: m.Mountpoint,
  236. major: uint(m.Major),
  237. minor: uint(m.Minor),
  238. }
  239. i.labels[LabelSystemRoot] = m.Source
  240. return
  241. }
  242. }
  243. }
  244. // addDockerImagesLabel attempts to determine which device contains the mount for docker images.
  245. func (i *RealFsInfo) addDockerImagesLabel(context Context, mounts []*mount.Info) {
  246. if context.Docker.Driver != "" {
  247. dockerDev, dockerPartition, err := i.getDockerDeviceMapperInfo(context.Docker)
  248. if err != nil {
  249. klog.Warningf("Could not get Docker devicemapper device: %v", err)
  250. }
  251. if len(dockerDev) > 0 && dockerPartition != nil {
  252. i.partitions[dockerDev] = *dockerPartition
  253. i.labels[LabelDockerImages] = dockerDev
  254. } else {
  255. i.updateContainerImagesPath(LabelDockerImages, mounts, getDockerImagePaths(context))
  256. }
  257. }
  258. }
  259. func (i *RealFsInfo) addCrioImagesLabel(context Context, mounts []*mount.Info) {
  260. if context.Crio.Root != "" {
  261. crioPath := context.Crio.Root
  262. crioImagePaths := map[string]struct{}{
  263. "/": {},
  264. }
  265. for _, dir := range []string{"devicemapper", "btrfs", "aufs", "overlay", "zfs"} {
  266. crioImagePaths[path.Join(crioPath, dir+"-images")] = struct{}{}
  267. }
  268. for crioPath != "/" && crioPath != "." {
  269. crioImagePaths[crioPath] = struct{}{}
  270. crioPath = filepath.Dir(crioPath)
  271. }
  272. i.updateContainerImagesPath(LabelCrioImages, mounts, crioImagePaths)
  273. }
  274. }
  275. // Generate a list of possible mount points for docker image management from the docker root directory.
  276. // Right now, we look for each type of supported graph driver directories, but we can do better by parsing
  277. // some of the context from `docker info`.
  278. func getDockerImagePaths(context Context) map[string]struct{} {
  279. dockerImagePaths := map[string]struct{}{
  280. "/": {},
  281. }
  282. // TODO(rjnagal): Detect docker root and graphdriver directories from docker info.
  283. dockerRoot := context.Docker.Root
  284. for _, dir := range []string{"devicemapper", "btrfs", "aufs", "overlay", "overlay2", "zfs"} {
  285. dockerImagePaths[path.Join(dockerRoot, dir)] = struct{}{}
  286. }
  287. for dockerRoot != "/" && dockerRoot != "." {
  288. dockerImagePaths[dockerRoot] = struct{}{}
  289. dockerRoot = filepath.Dir(dockerRoot)
  290. }
  291. return dockerImagePaths
  292. }
  293. // This method compares the mountpoints with possible container image mount points. If a match is found,
  294. // the label is added to the partition.
  295. func (i *RealFsInfo) updateContainerImagesPath(label string, mounts []*mount.Info, containerImagePaths map[string]struct{}) {
  296. var useMount *mount.Info
  297. for _, m := range mounts {
  298. if _, ok := containerImagePaths[m.Mountpoint]; ok {
  299. if useMount == nil || (len(useMount.Mountpoint) < len(m.Mountpoint)) {
  300. useMount = m
  301. }
  302. }
  303. }
  304. if useMount != nil {
  305. i.partitions[useMount.Source] = partition{
  306. fsType: useMount.FSType,
  307. mountpoint: useMount.Mountpoint,
  308. major: uint(useMount.Major),
  309. minor: uint(useMount.Minor),
  310. }
  311. i.labels[label] = useMount.Source
  312. }
  313. }
  314. func (i *RealFsInfo) GetDeviceForLabel(label string) (string, error) {
  315. dev, ok := i.labels[label]
  316. if !ok {
  317. return "", fmt.Errorf("non-existent label %q", label)
  318. }
  319. return dev, nil
  320. }
  321. func (i *RealFsInfo) GetLabelsForDevice(device string) ([]string, error) {
  322. var labels []string
  323. for label, dev := range i.labels {
  324. if dev == device {
  325. labels = append(labels, label)
  326. }
  327. }
  328. return labels, nil
  329. }
  330. func (i *RealFsInfo) GetMountpointForDevice(dev string) (string, error) {
  331. p, ok := i.partitions[dev]
  332. if !ok {
  333. return "", fmt.Errorf("no partition info for device %q", dev)
  334. }
  335. return p.mountpoint, nil
  336. }
  337. func (i *RealFsInfo) GetFsInfoForPath(mountSet map[string]struct{}) ([]Fs, error) {
  338. filesystems := make([]Fs, 0)
  339. deviceSet := make(map[string]struct{})
  340. diskStatsMap, err := getDiskStatsMap("/proc/diskstats")
  341. if err != nil {
  342. return nil, err
  343. }
  344. for device, partition := range i.partitions {
  345. _, hasMount := mountSet[partition.mountpoint]
  346. _, hasDevice := deviceSet[device]
  347. if mountSet == nil || (hasMount && !hasDevice) {
  348. var (
  349. err error
  350. fs Fs
  351. )
  352. switch partition.fsType {
  353. case DeviceMapper.String():
  354. fs.Capacity, fs.Free, fs.Available, err = getDMStats(device, partition.blockSize)
  355. klog.V(5).Infof("got devicemapper fs capacity stats: capacity: %v free: %v available: %v:", fs.Capacity, fs.Free, fs.Available)
  356. fs.Type = DeviceMapper
  357. case ZFS.String():
  358. if _, devzfs := os.Stat("/dev/zfs"); os.IsExist(devzfs) {
  359. fs.Capacity, fs.Free, fs.Available, err = getZfstats(device)
  360. fs.Type = ZFS
  361. break
  362. }
  363. // if /dev/zfs is not present default to VFS
  364. fallthrough
  365. default:
  366. var inodes, inodesFree uint64
  367. if utils.FileExists(partition.mountpoint) {
  368. fs.Capacity, fs.Free, fs.Available, inodes, inodesFree, err = getVfsStats(partition.mountpoint)
  369. fs.Inodes = &inodes
  370. fs.InodesFree = &inodesFree
  371. fs.Type = VFS
  372. } else {
  373. klog.V(4).Infof("unable to determine file system type, partition mountpoint does not exist: %v", partition.mountpoint)
  374. }
  375. }
  376. if err != nil {
  377. klog.V(4).Infof("Stat fs failed. Error: %v", err)
  378. } else {
  379. deviceSet[device] = struct{}{}
  380. fs.DeviceInfo = DeviceInfo{
  381. Device: device,
  382. Major: uint(partition.major),
  383. Minor: uint(partition.minor),
  384. }
  385. if val, ok := diskStatsMap[device]; ok {
  386. fs.DiskStats = val
  387. } else {
  388. for k, v := range diskStatsMap {
  389. if v.MajorNum == uint64(partition.major) && v.MinorNum == uint64(partition.minor) {
  390. fs.DiskStats = diskStatsMap[k]
  391. break
  392. }
  393. }
  394. }
  395. filesystems = append(filesystems, fs)
  396. }
  397. }
  398. }
  399. return filesystems, nil
  400. }
  401. var partitionRegex = regexp.MustCompile(`^(?:(?:s|v|xv)d[a-z]+\d*|dm-\d+)$`)
  402. func getDiskStatsMap(diskStatsFile string) (map[string]DiskStats, error) {
  403. diskStatsMap := make(map[string]DiskStats)
  404. file, err := os.Open(diskStatsFile)
  405. if err != nil {
  406. if os.IsNotExist(err) {
  407. klog.Warningf("Not collecting filesystem statistics because file %q was not found", diskStatsFile)
  408. return diskStatsMap, nil
  409. }
  410. return nil, err
  411. }
  412. defer file.Close()
  413. scanner := bufio.NewScanner(file)
  414. for scanner.Scan() {
  415. line := scanner.Text()
  416. words := strings.Fields(line)
  417. if !partitionRegex.MatchString(words[2]) {
  418. continue
  419. }
  420. // 8 50 sdd2 40 0 280 223 7 0 22 108 0 330 330
  421. deviceName := path.Join("/dev", words[2])
  422. var err error
  423. devInfo := make([]uint64, 2)
  424. for i := 0; i < len(devInfo); i++ {
  425. devInfo[i], err = strconv.ParseUint(words[i], 10, 64)
  426. if err != nil {
  427. return nil, err
  428. }
  429. }
  430. wordLength := len(words)
  431. offset := 3
  432. var stats = make([]uint64, wordLength-offset)
  433. if len(stats) < 11 {
  434. return nil, fmt.Errorf("could not parse all 11 columns of /proc/diskstats")
  435. }
  436. for i := offset; i < wordLength; i++ {
  437. stats[i-offset], err = strconv.ParseUint(words[i], 10, 64)
  438. if err != nil {
  439. return nil, err
  440. }
  441. }
  442. major64, err := strconv.ParseUint(words[0], 10, 64)
  443. if err != nil {
  444. return nil, err
  445. }
  446. minor64, err := strconv.ParseUint(words[1], 10, 64)
  447. if err != nil {
  448. return nil, err
  449. }
  450. diskStats := DiskStats{
  451. MajorNum: devInfo[0],
  452. MinorNum: devInfo[1],
  453. ReadsCompleted: stats[0],
  454. ReadsMerged: stats[1],
  455. SectorsRead: stats[2],
  456. ReadTime: stats[3],
  457. WritesCompleted: stats[4],
  458. WritesMerged: stats[5],
  459. SectorsWritten: stats[6],
  460. WriteTime: stats[7],
  461. IoInProgress: stats[8],
  462. IoTime: stats[9],
  463. WeightedIoTime: stats[10],
  464. Major: major64,
  465. Minor: minor64,
  466. }
  467. diskStatsMap[deviceName] = diskStats
  468. }
  469. return diskStatsMap, nil
  470. }
  471. func (i *RealFsInfo) GetGlobalFsInfo() ([]Fs, error) {
  472. return i.GetFsInfoForPath(nil)
  473. }
  474. func major(devNumber uint64) uint {
  475. return uint((devNumber >> 8) & 0xfff)
  476. }
  477. func minor(devNumber uint64) uint {
  478. return uint((devNumber & 0xff) | ((devNumber >> 12) & 0xfff00))
  479. }
  480. func (i *RealFsInfo) GetDeviceInfoByFsUUID(uuid string) (*DeviceInfo, error) {
  481. deviceName, found := i.fsUUIDToDeviceName[uuid]
  482. if !found {
  483. return nil, ErrNoSuchDevice
  484. }
  485. p, found := i.partitions[deviceName]
  486. if !found {
  487. return nil, fmt.Errorf("cannot find device %q in partitions", deviceName)
  488. }
  489. return &DeviceInfo{deviceName, p.major, p.minor}, nil
  490. }
  491. func (i *RealFsInfo) mountInfoFromDir(dir string) (*mount.Info, bool) {
  492. mnt, found := i.mounts[dir]
  493. // try the parent dir if not found until we reach the root dir
  494. // this is an issue on btrfs systems where the directory is not
  495. // the subvolume
  496. for !found {
  497. pathdir, _ := filepath.Split(dir)
  498. // break when we reach root
  499. if pathdir == "/" {
  500. mnt, found = i.mounts["/"]
  501. break
  502. }
  503. // trim "/" from the new parent path otherwise the next possible
  504. // filepath.Split in the loop will not split the string any further
  505. dir = strings.TrimSuffix(pathdir, "/")
  506. mnt, found = i.mounts[dir]
  507. }
  508. return &mnt, found
  509. }
  510. func (i *RealFsInfo) GetDirFsDevice(dir string) (*DeviceInfo, error) {
  511. buf := new(syscall.Stat_t)
  512. err := syscall.Stat(dir, buf)
  513. if err != nil {
  514. return nil, fmt.Errorf("stat failed on %s with error: %s", dir, err)
  515. }
  516. // The type Dev in Stat_t is 32bit on mips.
  517. major := major(uint64(buf.Dev)) // nolint: unconvert
  518. minor := minor(uint64(buf.Dev)) // nolint: unconvert
  519. for device, partition := range i.partitions {
  520. if partition.major == major && partition.minor == minor {
  521. return &DeviceInfo{device, major, minor}, nil
  522. }
  523. }
  524. mnt, found := i.mountInfoFromDir(dir)
  525. if found && strings.HasPrefix(mnt.Source, "/dev/") {
  526. major, minor := mnt.Major, mnt.Minor
  527. if mnt.FSType == "btrfs" && major == 0 {
  528. major, minor, err = getBtrfsMajorMinorIds(mnt)
  529. if err != nil {
  530. klog.Warningf("Unable to get btrfs mountpoint IDs: %v", err)
  531. }
  532. }
  533. return &DeviceInfo{mnt.Source, uint(major), uint(minor)}, nil
  534. }
  535. return nil, fmt.Errorf("with major: %d, minor: %d: %w", major, minor, ErrDeviceNotInPartitionsMap)
  536. }
  537. func GetDirUsage(dir string) (UsageInfo, error) {
  538. var usage UsageInfo
  539. if dir == "" {
  540. return usage, fmt.Errorf("invalid directory")
  541. }
  542. rootInfo, err := os.Stat(dir)
  543. if err != nil {
  544. return usage, fmt.Errorf("could not stat %q to get inode usage: %v", dir, err)
  545. }
  546. rootStat, ok := rootInfo.Sys().(*syscall.Stat_t)
  547. if !ok {
  548. return usage, fmt.Errorf("unsuported fileinfo for getting inode usage of %q", dir)
  549. }
  550. rootDevID := rootStat.Dev
  551. // dedupedInode stores inodes that could be duplicates (nlink > 1)
  552. dedupedInodes := make(map[uint64]struct{})
  553. err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
  554. if os.IsNotExist(err) {
  555. // expected if files appear/vanish
  556. return nil
  557. }
  558. if err != nil {
  559. return fmt.Errorf("unable to count inodes for part of dir %s: %s", dir, err)
  560. }
  561. // according to the docs, Sys can be nil
  562. if info.Sys() == nil {
  563. return fmt.Errorf("fileinfo Sys is nil")
  564. }
  565. s, ok := info.Sys().(*syscall.Stat_t)
  566. if !ok {
  567. return fmt.Errorf("unsupported fileinfo; could not convert to stat_t")
  568. }
  569. if s.Dev != rootDevID {
  570. // don't descend into directories on other devices
  571. return filepath.SkipDir
  572. }
  573. if s.Nlink > 1 {
  574. if _, ok := dedupedInodes[s.Ino]; !ok {
  575. // Dedupe things that could be hardlinks
  576. dedupedInodes[s.Ino] = struct{}{}
  577. usage.Bytes += uint64(s.Blocks) * statBlockSize
  578. usage.Inodes++
  579. }
  580. } else {
  581. usage.Bytes += uint64(s.Blocks) * statBlockSize
  582. usage.Inodes++
  583. }
  584. return nil
  585. })
  586. return usage, err
  587. }
  588. func (i *RealFsInfo) GetDirUsage(dir string) (UsageInfo, error) {
  589. claimToken()
  590. defer releaseToken()
  591. return GetDirUsage(dir)
  592. }
  593. func getVfsStats(path string) (total uint64, free uint64, avail uint64, inodes uint64, inodesFree uint64, err error) {
  594. var s syscall.Statfs_t
  595. if err = syscall.Statfs(path, &s); err != nil {
  596. return 0, 0, 0, 0, 0, err
  597. }
  598. total = uint64(s.Frsize) * s.Blocks
  599. free = uint64(s.Frsize) * s.Bfree
  600. avail = uint64(s.Frsize) * s.Bavail
  601. inodes = uint64(s.Files)
  602. inodesFree = uint64(s.Ffree)
  603. return total, free, avail, inodes, inodesFree, nil
  604. }
  605. // Devicemapper thin provisioning is detailed at
  606. // https://www.kernel.org/doc/Documentation/device-mapper/thin-provisioning.txt
  607. func dockerDMDevice(driverStatus map[string]string, dmsetup devicemapper.DmsetupClient) (string, uint, uint, uint, error) {
  608. poolName, ok := driverStatus[DriverStatusPoolName]
  609. if !ok || len(poolName) == 0 {
  610. return "", 0, 0, 0, fmt.Errorf("Could not get dm pool name")
  611. }
  612. out, err := dmsetup.Table(poolName)
  613. if err != nil {
  614. return "", 0, 0, 0, err
  615. }
  616. major, minor, dataBlkSize, err := parseDMTable(string(out))
  617. if err != nil {
  618. return "", 0, 0, 0, err
  619. }
  620. return poolName, major, minor, dataBlkSize, nil
  621. }
  622. // parseDMTable parses a single line of `dmsetup table` output and returns the
  623. // major device, minor device, block size, and an error.
  624. func parseDMTable(dmTable string) (uint, uint, uint, error) {
  625. dmTable = strings.Replace(dmTable, ":", " ", -1)
  626. dmFields := strings.Fields(dmTable)
  627. if len(dmFields) < 8 {
  628. return 0, 0, 0, fmt.Errorf("Invalid dmsetup status output: %s", dmTable)
  629. }
  630. major, err := strconv.ParseUint(dmFields[5], 10, 32)
  631. if err != nil {
  632. return 0, 0, 0, err
  633. }
  634. minor, err := strconv.ParseUint(dmFields[6], 10, 32)
  635. if err != nil {
  636. return 0, 0, 0, err
  637. }
  638. dataBlkSize, err := strconv.ParseUint(dmFields[7], 10, 32)
  639. if err != nil {
  640. return 0, 0, 0, err
  641. }
  642. return uint(major), uint(minor), uint(dataBlkSize), nil
  643. }
  644. func getDMStats(poolName string, dataBlkSize uint) (uint64, uint64, uint64, error) {
  645. out, err := exec.Command("dmsetup", "status", poolName).Output()
  646. if err != nil {
  647. return 0, 0, 0, err
  648. }
  649. used, total, err := parseDMStatus(string(out))
  650. if err != nil {
  651. return 0, 0, 0, err
  652. }
  653. used *= 512 * uint64(dataBlkSize)
  654. total *= 512 * uint64(dataBlkSize)
  655. free := total - used
  656. return total, free, free, nil
  657. }
  658. func parseDMStatus(dmStatus string) (uint64, uint64, error) {
  659. dmStatus = strings.Replace(dmStatus, "/", " ", -1)
  660. dmFields := strings.Fields(dmStatus)
  661. if len(dmFields) < 8 {
  662. return 0, 0, fmt.Errorf("Invalid dmsetup status output: %s", dmStatus)
  663. }
  664. used, err := strconv.ParseUint(dmFields[6], 10, 64)
  665. if err != nil {
  666. return 0, 0, err
  667. }
  668. total, err := strconv.ParseUint(dmFields[7], 10, 64)
  669. if err != nil {
  670. return 0, 0, err
  671. }
  672. return used, total, nil
  673. }
  674. // getZfstats returns ZFS mount stats using zfsutils
  675. func getZfstats(poolName string) (uint64, uint64, uint64, error) {
  676. dataset, err := zfs.GetDataset(poolName)
  677. if err != nil {
  678. return 0, 0, 0, err
  679. }
  680. total := dataset.Used + dataset.Avail + dataset.Usedbydataset
  681. return total, dataset.Avail, dataset.Avail, nil
  682. }
  683. // Get major and minor Ids for a mount point using btrfs as filesystem.
  684. func getBtrfsMajorMinorIds(mount *mount.Info) (int, int, error) {
  685. // btrfs fix: following workaround fixes wrong btrfs Major and Minor Ids reported in /proc/self/mountinfo.
  686. // instead of using values from /proc/self/mountinfo we use stat to get Ids from btrfs mount point
  687. buf := new(syscall.Stat_t)
  688. err := syscall.Stat(mount.Source, buf)
  689. if err != nil {
  690. err = fmt.Errorf("stat failed on %s with error: %s", mount.Source, err)
  691. return 0, 0, err
  692. }
  693. klog.V(4).Infof("btrfs mount %#v", mount)
  694. if buf.Mode&syscall.S_IFMT == syscall.S_IFBLK {
  695. err := syscall.Stat(mount.Mountpoint, buf)
  696. if err != nil {
  697. err = fmt.Errorf("stat failed on %s with error: %s", mount.Mountpoint, err)
  698. return 0, 0, err
  699. }
  700. // The type Dev and Rdev in Stat_t are 32bit on mips.
  701. klog.V(4).Infof("btrfs dev major:minor %d:%d\n", int(major(uint64(buf.Dev))), int(minor(uint64(buf.Dev)))) // nolint: unconvert
  702. klog.V(4).Infof("btrfs rdev major:minor %d:%d\n", int(major(uint64(buf.Rdev))), int(minor(uint64(buf.Rdev)))) // nolint: unconvert
  703. return int(major(uint64(buf.Dev))), int(minor(uint64(buf.Dev))), nil // nolint: unconvert
  704. }
  705. return 0, 0, fmt.Errorf("%s is not a block device", mount.Source)
  706. }