// Copyright 2019 Yunion // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package stats import ( "fmt" "path" "sort" "strings" cadvisorapiv2 "github.com/google/cadvisor/info/v2" "k8s.io/apimachinery/pkg/types" "k8s.io/klog/v2" "yunion.io/x/log" "yunion.io/x/onecloud/pkg/util/pod/cadvisor" ) // containerID is the identity of a container in a pod. type containerID struct { podRef PodReference containerName string } // buildPodRef returns a PodReference that identifies the Pod managing cinfo func buildPodRef(containerLabels map[string]string) PodReference { podName := GetPodName(containerLabels) podNamespace := GetPodNamespace(containerLabels) podUID := GetPodUID(containerLabels) return PodReference{Name: podName, Namespace: podNamespace, UID: podUID} } func getCadvisorContainerInfo(ca cadvisor.Interface) (map[string]cadvisorapiv2.ContainerInfo, error) { infos, err := ca.ContainerInfoV2("/", cadvisorapiv2.RequestOptions{ IdType: cadvisorapiv2.TypeName, Count: 2, // 2 samples are needed to compute "instantaneous" CPU Recursive: true, }) if err != nil { if _, ok := infos["/"]; ok { // If the failure is partial, log it and return a best-effort // response. log.Errorf("Partial failure issuing cadvisor.ContainerInfoV2: %v", err) } else { return nil, fmt.Errorf("failed to get root cgroup stats: %v", err) } } return infos, nil } // libcontainerCgroupManagerType defines how to interface with libcontainer type libcontainerCgroupManagerType string const ( // libcontainerCgroupfs means use libcontainer with cgroupfs libcontainerCgroupfs libcontainerCgroupManagerType = "cgroupfs" // libcontainerSystemd means use libcontainer with systemd libcontainerSystemd libcontainerCgroupManagerType = "systemd" // systemdSuffix is the cgroup name suffix for systemd systemdSuffix string = ".slice" ) func IsSystemdStyleName(name string) bool { return strings.HasSuffix(name, systemdSuffix) } // CgroupName is the abstract name of a cgroup prior to any driver specific conversion. // It is specified as a list of strings from its individual components, such as: // {"kubepods", "burstable", "pod1234-abcd-5678-efgh"} type CgroupName []string func ParseCgroupfsToCgroupName(name string) CgroupName { components := strings.Split(strings.TrimPrefix(name, "/"), "/") if len(components) == 1 && components[0] == "" { components = []string{} } return CgroupName(components) } func unescapeSystemdCgroupName(part string) string { return strings.Replace(part, "_", "-", -1) } func ParseSystemdToCgroupName(name string) CgroupName { driverName := path.Base(name) driverName = strings.TrimSuffix(driverName, systemdSuffix) parts := strings.Split(driverName, "-") result := []string{} for _, part := range parts { result = append(result, unescapeSystemdCgroupName(part)) } return CgroupName(result) } const ( podCgroupNamePrefix = "pod" ) // GetPodCgroupNameSuffix returns the last element of the pod CgroupName identifier func GetPodCgroupNameSuffix(podUID types.UID) string { return podCgroupNamePrefix + string(podUID) } func getContainerInfoById(id string, infos map[string]cadvisorapiv2.ContainerInfo) *cadvisorapiv2.ContainerInfo { for key, info := range infos { if IsSystemdStyleName(key) { // Convert to internal cgroup name and take the last component only. internalCgroupName := ParseSystemdToCgroupName(key) key = internalCgroupName[len(internalCgroupName)-1] } else { // Take last component only. key = path.Base(key) } //if GetPodCgroupNameSuffix(podUID) == key { if id == key { return &info } } return nil } func getLatestContainerStatsById(id string, infos map[string]cadvisorapiv2.ContainerInfo) *cadvisorapiv2.ContainerStats { info := getContainerInfoById(id, infos) if info == nil { return nil } cstat, found := latestContainerStats(info) if !found { return nil } return cstat } // getCadvisorPodInfoFromPodUID returns a pod cgroup information by matching the podUID with its CgroupName identifier base name func getCadvisorPodInfoFromPodUID(podUID types.UID, infos map[string]cadvisorapiv2.ContainerInfo) *cadvisorapiv2.ContainerInfo { return getContainerInfoById(string(podUID), infos) } // containerInfoWithCgroup contains the ContainerInfo and its cgroup name. type containerInfoWithCgroup struct { cinfo cadvisorapiv2.ContainerInfo cgroup string } // removeTerminatedContainerInfo returns the specified containerInfo but with // the stats of the terminated containers removed. // // A ContainerInfo is considered to be of a terminated container if it has an // older CreationTime and zero CPU instantaneous and memory RSS usage. func removeTerminatedContainerInfo(containerInfo map[string]cadvisorapiv2.ContainerInfo) map[string]cadvisorapiv2.ContainerInfo { cinfoMap := make(map[containerID][]containerInfoWithCgroup) for key, cinfo := range containerInfo { if !isPodManagedContainer(&cinfo) { continue } cinfoID := containerID{ podRef: buildPodRef(cinfo.Spec.Labels), containerName: GetContainerName(cinfo.Spec.Labels), } cinfoMap[cinfoID] = append(cinfoMap[cinfoID], containerInfoWithCgroup{ cinfo: cinfo, cgroup: key, }) } result := make(map[string]cadvisorapiv2.ContainerInfo) for _, refs := range cinfoMap { if len(refs) == 1 { result[refs[0].cgroup] = refs[0].cinfo continue } sort.Sort(ByCreationTime(refs)) for i := len(refs) - 1; i >= 0; i-- { if hasMemoryAndCPUInstUsage(&refs[i].cinfo) { result[refs[i].cgroup] = refs[i].cinfo break } } } return result } // hasMemoryAndCPUInstUsage returns true if the specified container info has // both non-zero CPU instantaneous usage and non-zero memory RSS usage, and // false otherwise. func hasMemoryAndCPUInstUsage(info *cadvisorapiv2.ContainerInfo) bool { if !info.Spec.HasCpu || !info.Spec.HasMemory { return false } cstat, found := latestContainerStats(info) if !found { return false } if cstat.CpuInst == nil { return false } return cstat.CpuInst.Usage.Total != 0 && cstat.Memory.RSS != 0 } // ByCreationTime implements sort.Interface for []containerInfoWithCgroup based // on the cinfo.Spec.CreationTime field. type ByCreationTime []containerInfoWithCgroup func (a ByCreationTime) Len() int { return len(a) } func (a ByCreationTime) Swap(i, j int) { a[i], a[j] = a[j], a[i] } func (a ByCreationTime) Less(i, j int) bool { if a[i].cinfo.Spec.CreationTime.Equal(a[j].cinfo.Spec.CreationTime) { // There shouldn't be two containers with the same name and/or the same // creation time. However, to make the logic here robust, we break the // tie by moving the one without CPU instantaneous or memory RSS usage // to the beginning. return hasMemoryAndCPUInstUsage(&a[j].cinfo) } return a[i].cinfo.Spec.CreationTime.Before(a[j].cinfo.Spec.CreationTime) } // isPodManagedContainer returns true if the cinfo container is managed by a Pod func isPodManagedContainer(cinfo *cadvisorapiv2.ContainerInfo) bool { podName := GetPodName(cinfo.Spec.Labels) podNamespace := GetPodNamespace(cinfo.Spec.Labels) managed := podName != "" && podNamespace != "" if !managed && podName != podNamespace { klog.Warningf( "Expect container to have either both podName (%s) and podNamespace (%s) labels, or neither.", podName, podNamespace) } return managed }