Cloudpods/backend/pkg/util/cgrouputils/cgroupv1/cgrouputils.go

// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cgroupv1

import (
	"bufio"
	"fmt"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strings"

	"yunion.io/x/log"

	"yunion.io/x/onecloud/pkg/util/cgrouputils/cgroup"
	"yunion.io/x/onecloud/pkg/util/fileutils2"
	"yunion.io/x/onecloud/pkg/util/procutils"
)

const (
	CGROUP_TASKS = "tasks"

	maxWeight     = 16
	normalizeBase = 1024
)

type CGroupTask struct {
	pid       string
	threadIds []string
	name      string
	weight    float64

	hand cgroup.ICGroupTask
}

func NewCGroupTask(pid, name string, cpuShares int, threadIds []string) *CGroupTask {
	return &CGroupTask{
		pid:       pid,
		name:      name,
		weight:    float64(cpuShares) / normalizeBase,
		threadIds: threadIds,
	}
}

func (*CGroupTask) Init() bool {
	if !manager.CgroupIsMounted() {
		if !fileutils2.Exists(manager.GetCgroupPath()) {
			if err := procutils.NewCommand("mkdir", "-p", manager.GetCgroupPath()).Run(); err != nil {
				log.Errorf("mkdir -p %s error: %v", manager.GetCgroupPath(), err)
			}
		}
		if err := procutils.NewCommand("mount", "-t", "tmpfs", "-o", "uid=0,gid=0,mode=0755",
			"cgroup", manager.GetCgroupPath()).Run(); err != nil {
			log.Errorf("mount cgroups path %s, error: %v", manager.GetCgroupPath(), err)
			return false
		}
	}
	file, err := os.Open("/proc/cgroups")
	if err != nil {
		log.Errorln(err)
		return false
	}
	defer file.Close()

	re := regexp.MustCompile(`\s+`)
	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		// #subsys_name    hierarchy       num_cgroups     enabled
		line := scanner.Text()
		if line[0] != '#' {
			parts := re.Split(line, -1)
			module := parts[0]
			if parts[3] == "0" { // disabled
				continue
			}
			if !manager.ModuleIsMounted(module) {
				moduleDir := path.Join(manager.GetCgroupPath(), module)
				if !fileutils2.Exists(moduleDir) {
					if output, err := procutils.NewCommand("mkdir", moduleDir).Output(); err != nil {
						log.Errorf("mkdir %s failed: %s, %s", moduleDir, err, output)
						return false
					}
				}
				if err := procutils.NewCommand("mount", "-t", "cgroup", "-o",
					module, module, moduleDir).Run(); err != nil {
					log.Errorf("mount cgroup module %s to %s error: %v", module, moduleDir, err)
					return false
				}
			}
		}
	}

	if err := scanner.Err(); err != nil {
		log.Errorf("scan file %s error: %v", file.Name(), err)
		return false
	}

	return true
}

func (c *CGroupTask) CustomConfig(key, value string) bool {
	configPath := GetTaskParamPath(c.hand.Module(), key, c.GroupName())
	if !fileutils2.Exists(configPath) {
		return true
	}
	return SetRootParam(c.hand.Module(), key, value, c.GroupName())
}

func (c *CGroupTask) SetWeight(coreNum int) {
	c.weight = float64(coreNum) / normalizeBase
}

func (c *CGroupTask) SetHand(hand cgroup.ICGroupTask) {
	c.hand = hand
}

func (c *CGroupTask) SetPid(pid string) {
	c.pid = pid
}

func (c *CGroupTask) SetName(name string) {
	c.name = name
}

func (c *CGroupTask) GroupName() string {
	if len(c.name) > 0 {
		return c.name
	}
	return c.pid
}

func (c *CGroupTask) InitTask(hand cgroup.ICGroupTask, coreNum int, pid, name string) {
	c.SetHand(hand)
	c.SetWeight(coreNum)
	c.SetPid(pid)
	c.SetName(name)
}

func (c *CGroupTask) Module() string {
	return ""
}

func (c *CGroupTask) GetWeight() float64 {
	if c.weight < maxWeight {
		return c.weight
	} else {
		return maxWeight
	}
}

func (c *CGroupTask) GetTaskIds() []string {
	if len(c.pid) == 0 {
		return nil
	}

	if len(c.threadIds) > 0 {
		return c.threadIds
	}

	files, err := ioutil.ReadDir(fmt.Sprintf("/proc/%s/task", c.pid))
	if err != nil {
		log.Errorf("GetTaskIds failed: %s", err)
		return nil
	}
	ids := []string{}
	for _, file := range files {
		ids = append(ids, file.Name())
	}
	return ids
}

func (c *CGroupTask) TaskPath() string {
	return path.Join(RootTaskPath(c.hand.Module()), c.GroupName())
}

func (c *CGroupTask) TaskIsExist() bool {
	return fileutils2.Exists(c.TaskPath())
}

func (c *CGroupTask) createTask() bool {
	if err := os.Mkdir(c.TaskPath(), os.ModePerm); err != nil {
		log.Errorln(err)
		return false
	}
	return true
}

func (c *CGroupTask) GetParam(name string) string {
	return GetRootParam(c.hand.Module(), name, c.GroupName())
}

func (c *CGroupTask) MoveTasksToRoot() {
	procs := c.GetParam(CGROUP_TASKS)
	if len(procs) > 0 {
		for _, proc := range strings.Split(procs, "\n") {
			proc = strings.TrimSpace(proc)
			if len(proc) > 0 {
				c.PushPid(proc, true)
			}
		}
	}
}

func (c *CGroupTask) RemoveTask() bool {
	if c.TaskIsExist() {
		c.MoveTasksToRoot()
		log.Infof("Remove task path %s %s", c.TaskPath(), c.name)
		if err := os.Remove(c.TaskPath()); err != nil {
			log.Errorf("Remove task path failed %s", err)
			return false
		}
	}
	return true
}

func (c *CGroupTask) GetStaticConfig() map[string]string {
	return nil
}

func (c *CGroupTask) GetConfig() map[string]string {
	return nil
}

func (c *CGroupTask) SetParam(name, value string) bool {
	return SetRootParam(c.hand.Module(), name, value, c.GroupName())
}

func (c *CGroupTask) SetParams(conf map[string]string) bool {
	for k, v := range conf {
		if !c.SetParam(k, v) {
			log.Errorf("Fail to set %s/%s=%s for %s", c.hand.Module(), k, v, c.GroupName())
			return false
		}
	}
	return true
}

func (c *CGroupTask) Configure() bool {
	if !c.TaskIsExist() {
		if !c.createTask() {
			return false
		}
	}

	conf := c.hand.GetStaticConfig()
	if !c.SetParams(conf) {
		return false
	}

	conf = c.hand.GetConfig()
	return c.SetParams(conf)
}

func (c *CGroupTask) SetTask() bool {
	if !c.TaskIsExist() {
		if !c.createTask() {
			return false
		}
	}

	conf := c.hand.GetStaticConfig()
	if !c.SetParams(conf) {
		return false
	}

	conf = c.hand.GetConfig()
	if c.SetParams(conf) {
		pids := c.GetTaskIds()
		if len(pids) > 0 {
			for _, pid := range pids {
				c.PushPid(pid, false)
			}
			return true
		}
	}
	return false
}

func (c *CGroupTask) PushPid(tid string, isRoot bool) {
	if c.pid == "" {
		return
	}

	subdir := fmt.Sprintf("/proc/%s/task/%s", c.pid, tid)
	if fi, err := os.Stat(subdir); err != nil {
		log.Errorf("Fail to put pid in task %s", err)
		return
	} else if fi.Mode().IsDir() {
		stat, err := fileutils2.FileGetContents(path.Join(subdir, "stat"))
		if err != nil {
			log.Errorf("Fail to put pid in task %s", err)
			return
		}
		re := regexp.MustCompile(`\s+`)
		data := re.Split(stat, -1)
		if data[2] != "Z" {
			if isRoot {
				SetRootParam(c.hand.Module(), CGROUP_TASKS, tid, "")
			} else {
				c.SetParam(CGROUP_TASKS, tid)
			}
		}
	}
}

/**
 *  CGroupCPUTask
 */

type CGroupCPUTask struct {
	*CGroupTask
}

const (
	CgroupsSharesWeight = 1024
	CPU_SHARES          = "cpu.shares"
)

func (c *CGroupCPUTask) Module() string {
	return "cpu"
}

func (c *CGroupCPUTask) GetConfig() map[string]string {
	wt := int(CgroupsSharesWeight * c.GetWeight())
	return map[string]string{CPU_SHARES: fmt.Sprintf("%d", wt)}
}

func (c *CGroupCPUTask) Init() bool {
	return SetRootParam(c.Module(), CPU_SHARES,
		fmt.Sprintf("%d", CgroupsSharesWeight), "")
}

func (m *cgroupManager) NewCGroupCPUTask(pid, name string, cpuShares int) cgroup.ICGroupTask {
	t := &CGroupCPUTask{NewCGroupTask(pid, name, cpuShares, nil)}
	t.SetHand(t)
	return t
}

/**
 *  CGroupIOTask
 */

type CGroupIOTask struct {
	*CGroupTask
}

const (
	IoWeightBase    = 100
	IoWeightMax     = 1000
	IoWeightMin     = 100
	BLOCK_IO_WEIGHT = "blkio.weight"

	BLOCK_IO_BFQ_WEIGHT = "blkio.bfq.weight"

	IOSCHED_CFQ = "cfq"
	IOSCHED_BFQ = "bfq"
)

func (c *CGroupIOTask) Module() string {
	return "blkio"
}

func (c *CGroupIOTask) GetConfig() map[string]string {
	wt := int(c.GetWeight() * IoWeightBase)
	if wt > IoWeightMax {
		wt = IoWeightMax
	} else if wt < IoWeightMin {
		wt = IoWeightMin
	}
	switch manager.GetIoScheduler() {
	case IOSCHED_CFQ:
		return map[string]string{BLOCK_IO_WEIGHT: fmt.Sprintf("%d", wt)}
	case IOSCHED_BFQ:
		return map[string]string{BLOCK_IO_BFQ_WEIGHT: fmt.Sprintf("%d", wt)}
	default:
		return nil
	}
}

func (c *CGroupIOTask) Init() bool {
	switch manager.GetIoScheduler() {
	case IOSCHED_CFQ:
		return SetRootParam(c.Module(), BLOCK_IO_WEIGHT, fmt.Sprintf("%d", IoWeightMax), "")
	default:
		return true
	}
}

func (m *cgroupManager) NewCGroupIOTask(pid, name string, cpuShares int) cgroup.ICGroupTask {
	task := &CGroupIOTask{NewCGroupTask(pid, name, cpuShares, nil)}
	task.SetHand(task)
	return task
}

/**
 *  CGroupIOHardlimitTask
 */

type CGroupIOHardlimitTask struct {
	*CGroupIOTask

	cpuNum int
	params map[string]int
	devId  string
}

func (c *CGroupIOHardlimitTask) GetConfig() map[string]string {
	config := make(map[string]string, 0)
	for k, v := range c.params {
		if v != 0 {
			config[k] = fmt.Sprintf("%s %d", c.devId, v*c.cpuNum)
		}
	}
	return config
}

func (m *cgroupManager) NewCGroupIOHardlimitTask(pid, name string, coreNum int, params map[string]int, devId string) cgroup.ICGroupTask {
	task := &CGroupIOHardlimitTask{
		CGroupIOTask: m.NewCGroupIOTask(pid, name, 0).(*CGroupIOTask),
		cpuNum:       coreNum,
		params:       params,
		devId:        devId,
	}
	task.SetHand(task)
	return task
}

/**
 *  CGroupMemoryTask
 */

type CGroupMemoryTask struct {
	*CGroupTask
}

const (
	root_swappiness   = 60
	vm_swappiness     = 0
	MEMORY_SWAPPINESS = "memory.swappiness"
)

func (c *CGroupMemoryTask) Module() string {
	return "memory"
}

func (c *CGroupMemoryTask) GetConfig() map[string]string {
	return map[string]string{MEMORY_SWAPPINESS: fmt.Sprintf("%d", vm_swappiness)}
}

func (m *cgroupManager) NewCGroupMemoryTask(pid, name string, coreNum int) cgroup.ICGroupTask {
	task := &CGroupMemoryTask{
		CGroupTask: NewCGroupTask(pid, name, coreNum, nil),
	}
	task.SetHand(task)
	return task
}

/**
 *  CGroupCPUSetTask
 */

type CGroupCPUSetTask struct {
	*CGroupTask

	cpuset string
	mems   string
}

const (
	CPUSET_CPUS = "cpuset.cpus"
	CPUSET_MEMS = "cpuset.mems"
)

func (c *CGroupCPUSetTask) Module() string {
	return "cpuset"
}

func (c *CGroupCPUSetTask) GetConfig() map[string]string {
	if c.cpuset == "" {
		parentPath := filepath.Dir(c.GroupName())
		c.cpuset = GetRootParam(c.Module(), CPUSET_CPUS, parentPath)
	}
	if c.mems == "" {
		parentPath := filepath.Dir(c.GroupName())
		c.mems = GetRootParam(c.Module(), CPUSET_MEMS, parentPath)
	}
	return map[string]string{CPUSET_CPUS: c.cpuset, CPUSET_MEMS: c.mems}
}

func (m *cgroupManager) NewCGroupCPUSetTask(pid, name, cpuset, mems string) cgroup.ICGroupTask {
	task := &CGroupCPUSetTask{
		CGroupTask: NewCGroupTask(pid, name, 0, nil),
		cpuset:     cpuset,
		mems:       mems,
	}
	task.SetHand(task)
	return task
}

func (m *cgroupManager) NewCGroupSubCPUSetTask(pid, name string, cpuset string, threadIds []string) cgroup.ICGroupTask {
	task := &CGroupCPUSetTask{
		CGroupTask: NewCGroupTask(pid, name, 0, threadIds),
		cpuset:     cpuset,
	}
	task.SetHand(task)
	return task
}