Cloudpods/backend/pkg/util/fileutils2/fileutils.go

// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fileutils2

import (
	"archive/tar"
	"bufio"
	"bytes"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"

	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"
	"yunion.io/x/pkg/util/regutils"

	"yunion.io/x/onecloud/pkg/util/procutils"
)

func Cleandir(sPath string, keepdir bool) error {
	if f, _ := os.Lstat(sPath); f == nil || f.Mode()&os.ModeSymlink == os.ModeSymlink {
		return nil
	}
	files, _ := ioutil.ReadDir(sPath)
	for _, file := range files {
		fp := path.Join(sPath, file.Name())
		if f, _ := os.Lstat(fp); f.Mode()&os.ModeSymlink == os.ModeSymlink {
			if !keepdir {
				if err := os.Remove(fp); err != nil {
					return err
				}
			}
		} else if f.IsDir() {
			Cleandir(fp, keepdir)
			if !keepdir {
				if err := os.Remove(fp); err != nil {
					return err
				}
			}
		} else {
			if err := os.Remove(fp); err != nil {
				return err
			}
		}
	}
	return nil
}

// TODO: test
func Zerofiles(sPath string) error {
	f, err := os.Lstat(sPath)
	switch {
	case err != nil:
		return err
	case f.Mode()&os.ModeSymlink == os.ModeSymlink:
		// islink
		return nil
	case f.Mode().IsRegular():
		return FilePutContents(sPath, "", false)
	case f.Mode().IsDir():
		files, err := ioutil.ReadDir(sPath)
		if err != nil {
			return err
		}
		for _, file := range files {
			if file.Mode()&os.ModeSymlink == os.ModeSymlink {
				continue
			} else if file.Mode().IsRegular() {
				if err := FilePutContents(path.Join(sPath, file.Name()), "", false); err != nil {
					return err
				}
			} else if file.Mode().IsDir() {
				return Zerofiles(path.Join(sPath, file.Name()))
			}
		}
	}
	return nil
}

func FileSetContents(filename string, content string) error {
	return FilePutContents(filename, content, false)
}

func FileAppendContents(filename string, content string) error {
	return FilePutContents(filename, content, true)
}

func FilePutContents(filename string, content string, modAppend bool) error {
	var mode = os.O_WRONLY | os.O_CREATE
	if modAppend {
		mode = mode | os.O_APPEND
	} else {
		mode = mode | os.O_TRUNC
	}
	fd, err := os.OpenFile(filename, mode, 0644)
	if err != nil {
		return err
	}
	defer fd.Close()
	_, err = fd.WriteString(content)
	return err
}

func IsBlockDevMounted(dev string) bool {
	devPath := "/dev/" + dev
	mounts, err := procutils.NewCommand("mount").Output()
	if err != nil {
		return false
	}
	for _, s := range strings.Split(string(mounts), "\n") {
		if strings.HasPrefix(s, devPath) {
			return true
		}
	}
	return false
}

func IsBlockDeviceUsed(dev string) bool {
	if strings.HasPrefix(dev, "/dev/") {
		dev = dev[strings.LastIndex(dev, "/")+1:]
	}
	devStr := fmt.Sprintf(" %s\n", dev)
	devs, _ := procutils.NewCommand("cat", "/proc/partitions").Output()
	if idx := strings.Index(string(devs), devStr); idx > 0 {
		return true
	}
	return false
}

func GetAllBlkdevsIoSchedulers() ([]string, error) {
	if _, err := os.Stat("/sys/block"); !os.IsNotExist(err) {
		blockDevs, err := os.ReadDir("/sys/block")
		if err != nil {
			log.Errorf("ReadDir /sys/block error: %s", err)
			return nil, errors.Wrap(err, "ioutil.ReadDir(/sys/block)")
		}
		for _, b := range blockDevs {
			// check is a block device
			if !Exists(path.Join("/sys/block", b.Name(), "device")) {
				continue
			}
			if IsBlockDevMounted(b.Name()) {
				conf, err := GetBlkdevConfig(b.Name(), "queue/scheduler")
				if err != nil {
					log.Errorf("Get %s queue/scheduler fail %s", b.Name(), err)
				} else {
					algs := make([]string, 0)
					for _, alg := range strings.Split(strings.TrimSpace(conf), " ") {
						if len(alg) > 0 {
							if alg[0] == '[' {
								alg = alg[1 : len(alg)-1]
							}
							algs = append(algs, alg)
						}
					}
					return algs, nil
				}
			}
		}
		log.Errorf("no block device avaiable")
		return nil, nil
	} else {
		log.Errorf("stat /sys/block fail %s", err)
		return nil, errors.Wrap(err, "stat /sys/block fail")
	}
}

func ChangeAllBlkdevsParams(params map[string]string) {
	if _, err := os.Stat("/sys/block"); !os.IsNotExist(err) {
		blockDevs, err := ioutil.ReadDir("/sys/block")
		if err != nil {
			log.Errorf("ReadDir /sys/block error: %s", err)
			return
		}
		for _, b := range blockDevs {
			if !Exists(path.Join("/sys/block", b.Name(), "device")) {
				continue
			}
			for k, v := range params {
				ChangeBlkdevParameter(b.Name(), k, v)
			}
		}
	}
}

func BlockDevIsSsd(dev string) bool {
	rotational := path.Join("/sys/block", dev, "queue", "rotational")
	res, err := FileGetContents(rotational)
	if err != nil {
		log.Errorf("FileGetContents fail %s %s", rotational, err)
		return false
	}
	return strings.TrimSpace(res) == "0"
}

func ChangeSsdBlkdevsParams(params map[string]string) {
	if _, err := os.Stat("/sys/block"); !os.IsNotExist(err) {
		blockDevs, err := os.ReadDir("/sys/block")
		if err != nil {
			log.Errorf("ReadDir /sys/block error: %s", err)
			return
		}
		for _, b := range blockDevs {
			if !Exists(path.Join("/sys/block", b.Name(), "device")) {
				continue
			}
			if !BlockDevIsSsd(b.Name()) {
				continue
			}
			for k, v := range params {
				ChangeBlkdevParameter(b.Name(), k, v)
			}
		}
	}
}

func ChangeHddBlkdevsParams(params map[string]string) {
	if _, err := os.Stat("/sys/block"); !os.IsNotExist(err) {
		blockDevs, err := os.ReadDir("/sys/block")
		if err != nil {
			log.Errorf("ReadDir /sys/block error: %s", err)
			return
		}
		for _, b := range blockDevs {
			if !Exists(path.Join("/sys/block", b.Name(), "device")) {
				continue
			}
			if BlockDevIsSsd(b.Name()) {
				continue
			}
			for k, v := range params {
				ChangeBlkdevParameter(b.Name(), k, v)
			}
		}
	}
}

func ChangeBlkdevParameter(dev, key, value string) {
	p := path.Join("/sys/block", dev, key)
	if _, err := os.Stat(p); !os.IsNotExist(err) {
		err = FilePutContents(p, value, false)
		if err != nil {
			log.Errorf("Fail to set %s of %s to %s:%s", key, dev, value, err)
		}
		log.Infof("Set %s of %s to %s", key, dev, value)
	}
}

func GetBlkdevConfig(dev, key string) (string, error) {
	p := path.Join("/sys/block", dev, key)
	if _, err := os.Stat(p); !os.IsNotExist(err) {
		return FileGetContents(p)
	} else {
		return "", err
	}
}

func FileGetContents(file string) (string, error) {
	content, err := ioutil.ReadFile(file)
	if err != nil {
		return "", err
	}
	return string(content), nil
}

func FileGetIntContent(file string) (int, error) {
	content, err := FileGetContents(file)
	if err != nil {
		return -1, errors.Wrap(err, "FileGetContents")
	}
	val, err := strconv.Atoi(strings.TrimSpace(content))
	if err != nil {
		return -1, errors.Wrapf(err, "convert %s to int", content)
	}
	return val, nil
}

func GetFsFormat(diskPath string) string {
	ret, err := procutils.NewCommand("blkid", "-o", "value", "-s", "TYPE", diskPath).Output()
	if err != nil {
		log.Errorf("failed exec blkid of dev %s: %s, %s", diskPath, err, ret)
		return ""
	}
	var res string
	for _, line := range strings.Split(string(ret), "\n") {
		res += line
	}
	return res
}

func CleanFailedMountpoints() {
	var mtfile = "/etc/mtab"
	if _, err := os.Stat(mtfile); os.IsNotExist(err) {
		mtfile = "/proc/mounts"
	}
	f, err := os.Open(mtfile)
	if err != nil {
		log.Errorf("CleanFailedMountpoints error: %s", err)
	}
	reader := bufio.NewReader(f)
	line, _, err := reader.ReadLine()
	for err != nil {
		m := strings.Split(string(line), " ")
		if len(m) > 1 {
			mp := m[1]
			if _, err := os.Stat(mp); os.IsNotExist(err) {
				log.Warningf("Mount point %s not exists", mp)
			}
			procutils.NewCommand("umount", mp).Run()
		}
	}
}

type HostsFile map[string][]string

func (hf HostsFile) Parse(content string) {
	lines := strings.Split(content, "\n")
	for _, line := range lines {
		data := regexp.MustCompile(`\s+`).Split(line, -1)
		for len(data) > 0 && data[len(data)-1] == "" {
			data = data[:len(data)-1]
		}
		if len(data) > 1 {
			hf[data[0]] = data[1:]
		}
	}
}

func (hf HostsFile) Add(name string, value ...string) {
	hf[name] = value
}

func (hf HostsFile) String() string {
	var ret = ""
	for k, v := range hf {
		if len(v) > 0 {
			ret += fmt.Sprintf("%s\t%s\n", k, strings.Join(v, "\t"))
		}
	}
	return ret
}

func FormatHostsFile(content string, ips []string, hostname, hostdomain string) string {
	hf := make(HostsFile, 0)
	hf.Parse(content)
	hf.Add("127.0.0.1", "localhost")
	isV6 := false
	for _, ip := range ips {
		if regutils.MatchIP6Addr(ip) {
			isV6 = true
		}
	}
	if isV6 {
		hf.Add("::1", "localhost", "ip6-localhost", "ip6-loopback")
	}
	for _, ip := range ips {
		hf.Add(ip, hostdomain, hostname)
	}
	return hf.String()
}

func FsFormatToDiskType(fsFormat string) string {
	switch {
	case fsFormat == "swap":
		return "linux-swap"
	case strings.HasPrefix(fsFormat, "ext") || fsFormat == "xfs" || fsFormat == "f2fs":
		return "ext2"
	case strings.HasPrefix(fsFormat, "fat"):
		return "fat32"
	case fsFormat == "ntfs":
		return fsFormat
	default:
		return ""
	}
}

func GetDevOfPath(spath string) string {
	spath, err := filepath.Abs(spath)
	if err != nil {
		log.Errorln(err)
		return ""
	}
	lines, err := procutils.NewCommand("mount").Output()
	if err != nil {
		log.Errorln(err)
		return ""
	}
	var (
		maxMatchLen int
		matchDev    string
	)

	for _, line := range strings.Split(string(lines), "\n") {
		segs := strings.Split(line, " ")
		if len(segs) < 3 {
			continue
		}
		if strings.HasPrefix(segs[0], "/dev/") {
			if strings.HasPrefix(spath, segs[2]) {
				matchLen := len(segs[2])
				if maxMatchLen < matchLen {
					maxMatchLen = matchLen
					matchDev = segs[0]
				}
			}
		}
	}
	return matchDev
}

func GetDevId(spath string) string {
	dev := GetDevOfPath(spath)
	if len(dev) == 0 {
		return ""
	}
	devInfo, err := procutils.NewCommand("ls", "-l", dev).Output()
	if err != nil {
		log.Errorln(err)
		return ""
	}
	devInfos := strings.Split(string(devInfo), "\n")
	data := strings.Split(string(devInfos[0]), " ")
	data[4] = data[4][:len(data[4])-1]
	return strings.Join(data, ":")
}

func GetDevUuid(dev string) (map[string]string, error) {
	lines, err := procutils.NewCommand("blkid", dev).Output()
	if err != nil {
		log.Errorf("GetDevUuid %s error: %v", dev, err)
		return map[string]string{}, errors.Wrapf(err, "blkid")
	}
	for _, l := range strings.Split(string(lines), "\n") {
		if strings.HasPrefix(l, dev) {
			var ret = map[string]string{}
			for _, part := range strings.Split(l, " ") {
				data := strings.Split(part, "=")
				if len(data) == 2 && strings.HasSuffix(data[0], "UUID") {
					if data[1][0] == '"' || data[1][0] == '\'' {
						ret[data[0]] = data[1][1 : len(data[1])-1]
					} else {
						ret[data[0]] = data[1]
					}
				}
			}
			return ret, nil
		}
	}
	return map[string]string{}, nil
}

func IsIsoFile(sPath string) bool {
	file, err := os.Open(sPath)
	if err != nil {
		return false
	}
	defer file.Close()
	file.Seek(0x8001, 0)
	buffer := make([]byte, 5)
	_, err = file.Read(buffer)
	if err != nil {
		return false
	}
	return bytes.Equal(buffer, []byte("CD001"))
}

func IsTarGzipFile(fPath string) bool {
	f, err := os.Open(fPath)
	if err != nil {
		return false
	}
	defer f.Close()

	gzf, err := gzip.NewReader(f)
	if err != nil {
		return false
	}

	return IsTarStream(gzf)
}

func IsTarStream(f io.Reader) bool {
	tarReader := tar.NewReader(f)
	_, err := tarReader.Next()
	if err != nil {
		return false
	}
	return true
}

func IsTarFile(fPath string) bool {
	f, err := os.Open(fPath)
	if err != nil {
		return false
	}
	defer f.Close()
	return IsTarStream(f)
}