oomparser.go 5.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174
  1. // Copyright 2014 Google Inc. All Rights Reserved.
  2. //
  3. // Licensed under the Apache License, Version 2.0 (the "License");
  4. // you may not use this file except in compliance with the License.
  5. // You may obtain a copy of the License at
  6. //
  7. // http://www.apache.org/licenses/LICENSE-2.0
  8. //
  9. // Unless required by applicable law or agreed to in writing, software
  10. // distributed under the License is distributed on an "AS IS" BASIS,
  11. // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  12. // See the License for the specific language governing permissions and
  13. // limitations under the License.
  14. package oomparser
  15. import (
  16. "path"
  17. "regexp"
  18. "strconv"
  19. "time"
  20. "github.com/euank/go-kmsg-parser/kmsgparser"
  21. "k8s.io/klog/v2"
  22. )
  23. var (
  24. legacyContainerRegexp = regexp.MustCompile(`Task in (.*) killed as a result of limit of (.*)`)
  25. // Starting in 5.0 linux kernels, the OOM message changed
  26. containerRegexp = regexp.MustCompile(`oom-kill:constraint=(.*),nodemask=(.*),cpuset=(.*),mems_allowed=(.*),oom_memcg=(.*),task_memcg=(.*),task=(.*),pid=(.*),uid=(.*)`)
  27. lastLineRegexp = regexp.MustCompile(`Killed process ([0-9]+) \((.+)\)`)
  28. firstLineRegexp = regexp.MustCompile(`invoked oom-killer:`)
  29. )
  30. // OomParser wraps a kmsgparser in order to extract OOM events from the
  31. // individual kernel ring buffer messages.
  32. type OomParser struct {
  33. parser kmsgparser.Parser
  34. }
  35. // struct that contains information related to an OOM kill instance
  36. type OomInstance struct {
  37. // process id of the killed process
  38. Pid int
  39. // the name of the killed process
  40. ProcessName string
  41. // the time that the process was reported to be killed,
  42. // accurate to the minute
  43. TimeOfDeath time.Time
  44. // the absolute name of the container that OOMed
  45. ContainerName string
  46. // the absolute name of the container that was killed
  47. // due to the OOM.
  48. VictimContainerName string
  49. // the constraint that triggered the OOM. One of CONSTRAINT_NONE,
  50. // CONSTRAINT_CPUSET, CONSTRAINT_MEMORY_POLICY, CONSTRAINT_MEMCG
  51. Constraint string
  52. }
  53. // gets the container name from a line and adds it to the oomInstance.
  54. func getLegacyContainerName(line string, currentOomInstance *OomInstance) error {
  55. parsedLine := legacyContainerRegexp.FindStringSubmatch(line)
  56. if parsedLine == nil {
  57. return nil
  58. }
  59. currentOomInstance.ContainerName = path.Join("/", parsedLine[1])
  60. currentOomInstance.VictimContainerName = path.Join("/", parsedLine[2])
  61. return nil
  62. }
  63. // gets the container name from a line and adds it to the oomInstance.
  64. func getContainerName(line string, currentOomInstance *OomInstance) (bool, error) {
  65. parsedLine := containerRegexp.FindStringSubmatch(line)
  66. if parsedLine == nil {
  67. // Fall back to the legacy format if it isn't found here.
  68. return false, getLegacyContainerName(line, currentOomInstance)
  69. }
  70. currentOomInstance.ContainerName = parsedLine[6]
  71. currentOomInstance.VictimContainerName = parsedLine[5]
  72. currentOomInstance.Constraint = parsedLine[1]
  73. pid, err := strconv.Atoi(parsedLine[8])
  74. if err != nil {
  75. return false, err
  76. }
  77. currentOomInstance.Pid = pid
  78. currentOomInstance.ProcessName = parsedLine[7]
  79. return true, nil
  80. }
  81. // gets the pid, name, and date from a line and adds it to oomInstance
  82. func getProcessNamePid(line string, currentOomInstance *OomInstance) (bool, error) {
  83. reList := lastLineRegexp.FindStringSubmatch(line)
  84. if reList == nil {
  85. return false, nil
  86. }
  87. pid, err := strconv.Atoi(reList[1])
  88. if err != nil {
  89. return false, err
  90. }
  91. currentOomInstance.Pid = pid
  92. currentOomInstance.ProcessName = reList[2]
  93. return true, nil
  94. }
  95. // uses regex to see if line is the start of a kernel oom log
  96. func checkIfStartOfOomMessages(line string) bool {
  97. potentialOomStart := firstLineRegexp.MatchString(line)
  98. return potentialOomStart
  99. }
  100. // StreamOoms writes to a provided a stream of OomInstance objects representing
  101. // OOM events that are found in the logs.
  102. // It will block and should be called from a goroutine.
  103. func (p *OomParser) StreamOoms(outStream chan<- *OomInstance) {
  104. kmsgEntries := p.parser.Parse()
  105. defer p.parser.Close()
  106. for msg := range kmsgEntries {
  107. isOomMessage := checkIfStartOfOomMessages(msg.Message)
  108. if isOomMessage {
  109. oomCurrentInstance := &OomInstance{
  110. ContainerName: "/",
  111. VictimContainerName: "/",
  112. TimeOfDeath: msg.Timestamp,
  113. }
  114. for msg := range kmsgEntries {
  115. finished, err := getContainerName(msg.Message, oomCurrentInstance)
  116. if err != nil {
  117. klog.Errorf("%v", err)
  118. }
  119. if !finished {
  120. finished, err = getProcessNamePid(msg.Message, oomCurrentInstance)
  121. if err != nil {
  122. klog.Errorf("%v", err)
  123. }
  124. }
  125. if finished {
  126. oomCurrentInstance.TimeOfDeath = msg.Timestamp
  127. break
  128. }
  129. }
  130. outStream <- oomCurrentInstance
  131. }
  132. }
  133. // Should not happen
  134. klog.Errorf("exiting analyzeLines. OOM events will not be reported.")
  135. }
  136. // initializes an OomParser object. Returns an OomParser object and an error.
  137. func New() (*OomParser, error) {
  138. parser, err := kmsgparser.NewParser()
  139. if err != nil {
  140. return nil, err
  141. }
  142. parser.SetLogger(glogAdapter{})
  143. return &OomParser{parser: parser}, nil
  144. }
  145. type glogAdapter struct{}
  146. var _ kmsgparser.Logger = glogAdapter{}
  147. func (glogAdapter) Infof(format string, args ...interface{}) {
  148. klog.V(4).Infof(format, args...)
  149. }
  150. func (glogAdapter) Warningf(format string, args ...interface{}) {
  151. klog.V(2).Infof(format, args...)
  152. }
  153. func (glogAdapter) Errorf(format string, args ...interface{}) {
  154. klog.Warningf(format, args...)
  155. }