Cloudpods/backend/pkg/hostman/guestman/libvirt.go

// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package guestman

import (
	"context"
	"encoding/xml"
	"fmt"
	"io/ioutil"
	"path"
	"strconv"
	"strings"
	"unicode"

	libvirtxml "github.com/libvirt/libvirt-go-xml"

	"yunion.io/x/jsonutils"
	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"
	"yunion.io/x/pkg/util/netutils"

	"yunion.io/x/onecloud/pkg/apis/compute"
	"yunion.io/x/onecloud/pkg/hostman/hostutils"
	"yunion.io/x/onecloud/pkg/hostman/storageman"
	"yunion.io/x/onecloud/pkg/util/fileutils2"
	"yunion.io/x/onecloud/pkg/util/procutils"
	"yunion.io/x/onecloud/pkg/util/qemuimg"
)

const defaultLibvirtLiveConfigPath = "/var/run/libvirt/qemu"

func (m *SGuestManager) GuestCreateFromLibvirt(
	ctx context.Context, params interface{},
) (jsonutils.JSONObject, error) {
	createConfig, ok := params.(*SGuestCreateFromLibvirt)
	if !ok {
		return nil, hostutils.ParamsError
	}
	disks := createConfig.GuestDesc.Disks

	disksPath := jsonutils.NewDict()
	for _, disk := range disks {
		diskPath, err := createConfig.DisksPath.GetString(disk.DiskId)
		if err != nil {
			return nil, fmt.Errorf("Disks path missing disk %s", disk.DiskId)
		}
		storage := storageman.GetManager().GetStorage(disk.StorageId)
		if storage == nil {
			return nil, fmt.Errorf("Host has no stroage %s", disk.StorageId)
		}
		iDisk := storage.CreateDisk(disk.DiskId)

		// use symbol link replace mv, more security
		output, err := procutils.NewCommand("ln", "-s", diskPath, iDisk.GetPath()).Output()
		if err != nil {
			return nil, fmt.Errorf("Symbol link disk from %s to %s error %s", diskPath, iDisk.GetPath(), output)
		}
		disksPath.Set(disk.DiskId, jsonutils.NewString(iDisk.GetPath()))
	}
	guest, _ := m.GetKVMServer(createConfig.Sid)
	if err := SaveDesc(guest, createConfig.GuestDesc); err != nil {
		return nil, err
	}

	if len(createConfig.MonitorPath) > 0 {
		if pid := findGuestProcessPid(guest.getOriginId()); len(pid) > 0 {
			fileutils2.FilePutContents(guest.GetPidFilePath(), pid, false)
			guest.StartMonitorWithImportGuestSocketFile(ctx, createConfig.MonitorPath, nil)
			stopScript := guest.generateStopScript(nil)
			if err := fileutils2.FilePutContents(guest.GetStopScriptPath(), stopScript, false); err != nil {
				return nil, fmt.Errorf("Save stop script error %s", err)
			}
		}
	}

	ret := jsonutils.NewDict()
	ret.Set("disks_path", disksPath)
	return ret, nil
}

func findGuestProcessPid(originId string) string {
	output, err := procutils.NewCommand(
		"sh", "-c", fmt.Sprintf("ps -A -o pid,args | grep [q]emu | grep %s", originId)).Output()
	if err != nil {
		log.Errorf("find guest %s error: %s", originId, output)
		return ""
	}
	var spid string
	s1 := strings.Split(strings.TrimSpace(string(output)), "\n")
	for i := 0; i < len(s1); i++ {
		if len(s1[i]) > 0 {
			if len(spid) > 0 {
				log.Errorf("can't find guest %s pid, has multi process", originId)
				return ""
			}
			s2 := strings.Fields(s1[i])
			if len(s2) > 1 {
				spid = s2[0]
			}
		}
	}
	if len(spid) > 0 {
		_, err := strconv.Atoi(spid)
		if err != nil {
			log.Errorf("Pid atoi failed %s", err)
			return ""
		} else {
			return spid
		}
	} else {
		return ""
	}
}

func (m *SGuestManager) PrepareImportFromLibvirt(
	ctx context.Context, params interface{},
) (jsonutils.JSONObject, error) {
	libvirtConfig, ok := params.(*compute.SLibvirtHostConfig)
	if !ok {
		return nil, hostutils.ParamsError
	}
	guestDescs, err := m.GenerateDescFromXml(libvirtConfig)
	if err != nil {
		return nil, err
	}
	return guestDescs, nil
}

func IsMacInGuestConfig(guestConfig *compute.SImportGuestDesc, mac string) bool {
	for _, nic := range guestConfig.Nics {
		if netutils.FormatMacAddr(nic.Mac) == netutils.FormatMacAddr(mac) {
			return true
		}
	}
	return false
}

func setAttributeFromLibvirtConfig(
	guestConfig *compute.SImportGuestDesc,
	libvirtConfig *compute.SLibvirtHostConfig,
	monitorPath string,
) (int, error) {
	var Matched = true
	for i, server := range libvirtConfig.Servers {
		macMap := make(map[string]string, 0)
		for mac := range server.MacIp {
			if !IsMacInGuestConfig(guestConfig, mac) {
				Matched = false
				break
			} else {
				macMap[netutils.FormatMacAddr(mac)] = server.MacIp[mac]
				Matched = true
			}
		}

		if Matched {
			for idx, nic := range guestConfig.Nics {
				guestConfig.Nics[idx].Ip = macMap[netutils.FormatMacAddr(nic.Mac)]
			}
			log.Infof("config monitor path is %s, guest config id %s", libvirtConfig.MonitorPath, guestConfig.Id)
			if len(monitorPath) > 0 {
				guestConfig.MonitorPath = monitorPath
			} else if len(libvirtConfig.MonitorPath) > 0 {
				files, _ := ioutil.ReadDir(libvirtConfig.MonitorPath)
				for i := 0; i < len(files); i++ {
					if files[i].Mode().IsDir() &&
						strings.HasPrefix(files[i].Name(), "domain") &&
						strings.HasSuffix(files[i].Name(), guestConfig.Name) {
						monitorPath := path.Join(libvirtConfig.MonitorPath, files[i].Name(), "monitor.sock")
						if fileutils2.Exists(monitorPath) && isServerRunning("[q]emu-kvm", guestConfig.Id) {
							guestConfig.MonitorPath = monitorPath
						}
						break
					}
				}
			}
			log.Infof("Import guest %s, monitor is %s", guestConfig.Name, guestConfig.MonitorPath)
			return i, nil
		}
	}
	return -1, fmt.Errorf("Config not match guest %s", guestConfig.Id)
}

func isServerRunning(sufix, uuid string) bool {
	return procutils.NewCommand("sh", "-c",
		fmt.Sprintf("ps -ef | grep [q]emu | grep %s | grep %s", uuid, sufix)).Run() == nil
}

func (m *SGuestManager) GenerateDescFromXml(libvirtConfig *compute.SLibvirtHostConfig) (jsonutils.JSONObject, error) {
	out, err := procutils.NewRemoteCommandAsFarAsPossible(
		"find", libvirtConfig.XmlFilePath,
		"-type", "f", "-maxdepth", "1",
	).Output()
	if err != nil {
		log.Errorf("failed read dir %s", libvirtConfig.XmlFilePath)
		return nil, errors.Wrapf(err, "failed read dir %s", libvirtConfig.XmlFilePath)
	}

	libvirtServers := []*compute.SImportGuestDesc{}
	xmlsPath := strings.Split(string(out), "\n")
	for _, xmlPath := range xmlsPath {
		if len(xmlPath) == 0 {
			continue
		}

		xmlContent, err := procutils.NewRemoteCommandAsFarAsPossible("cat", xmlPath).Output()
		if err != nil {
			log.Errorf("Read file %s failed: %s %s", xmlPath, xmlContent, err)
			continue
		}

		// parse libvirt xml file
		domain := &libvirtxml.Domain{}
		err = domain.Unmarshal(strings.TrimSpace(string(xmlContent)))
		if err != nil {
			log.Errorf("Unmarshal xml file %s error %s", xmlPath, err)
			continue
		}
		log.Infof("import domain uuid %s", domain.UUID)
		var monitorPath string
		if len(domain.UUID) > 0 {
			domainXmlFileName := fmt.Sprintf("%s.xml", domain.UUID)
			domainXmlFilePath := path.Join(defaultLibvirtLiveConfigPath, domainXmlFileName)
			if _, err := procutils.RemoteStat(domainXmlFilePath); err == nil {
				log.Infof("import domain live xml path %s", domainXmlFilePath)
				domStatus, err := m.getLibvirtLiveConfig(domainXmlFilePath)
				if err == nil {
					monitorPath = domStatus.Monitor.Path
				} else {
					log.Errorf("failed getLibvirtLiveConfig %s: %s", domainXmlFilePath, err)
				}
			}
		}

		guestConfig, err := m.LibvirtDomainToGuestDesc(domain)
		if err != nil {
			log.Errorf("Parse libvirt domain failed %s", err)
			continue
		}
		if idx, err := setAttributeFromLibvirtConfig(guestConfig, libvirtConfig, monitorPath); err != nil {
			log.Errorf("Import guest %s error %s", guestConfig.Id, err)
			continue
		} else {
			libvirtConfig.Servers = append(libvirtConfig.Servers[:idx], libvirtConfig.Servers[idx+1:]...)
			libvirtServers = append(libvirtServers, guestConfig)
		}
	}

	ret := jsonutils.NewDict()
	ret.Set("servers_not_match", jsonutils.Marshal(libvirtConfig.Servers))
	ret.Set("servers_matched", jsonutils.Marshal(libvirtServers))
	return ret, nil
}

type SMonitor struct {
	XMLName xml.Name `xml:"monitor"`
	Path    string   `xml:"path,attr"`
}

type SDomainStatus struct {
	XMLName xml.Name `xml:"domstatus"`
	Pid     int      `xml:"pid,attr"`
	Monitor SMonitor `xml:"monitor"`
}

func (m *SGuestManager) getLibvirtLiveConfig(domainXmlFilePath string) (*SDomainStatus, error) {
	liveXmlContent, err := procutils.NewRemoteCommandAsFarAsPossible("cat", domainXmlFilePath).Output()
	if err != nil {
		return nil, errors.Errorf("Read file %s failed: %s %s", domainXmlFilePath, liveXmlContent, err)
	}

	// guest running
	domainLive := &SDomainStatus{}
	err = xml.Unmarshal(liveXmlContent, domainLive)
	if err != nil {
		return nil, errors.Errorf("failed unmarshal live xml")
	}
	return domainLive, nil
}

// Read key infomation from domain xml
func (m *SGuestManager) LibvirtDomainToGuestDesc(domain *libvirtxml.Domain) (*compute.SImportGuestDesc, error) {
	if nil == domain {
		return nil, fmt.Errorf("Libvirt domain is nil")
	}
	if nil == domain.VCPU {
		return nil, fmt.Errorf("Libvirt domain missing VCPU config")
	}
	if nil == domain.CurrentMemory {
		return nil, fmt.Errorf("Libvirt domain missing CurrentMemory config")
	}
	if 0 == len(domain.CurrentMemory.Unit) {
		return nil, fmt.Errorf("Libvirt Memory config missing unit")
	}
	if nil == domain.Devices {
		return nil, fmt.Errorf("Libvirt domain missing Devices config")
	}
	if 0 == len(domain.Devices.Disks) {
		return nil, fmt.Errorf("Livbirt domain has no Disks")
	}
	if 0 == len(domain.Devices.Interfaces) {
		return nil, fmt.Errorf("Libvirt domain has no network Interfaces")
	}

	var memSizeMb uint
	switch unicode.ToLower(rune(domain.CurrentMemory.Unit[0])) {
	case 'k':
		memSizeMb = domain.CurrentMemory.Value / 1024
		if domain.CurrentMemory.Value%1024 > 0 {
			memSizeMb += 1
		}
	case 'm':
		memSizeMb = domain.CurrentMemory.Value
	case 'g':
		memSizeMb = domain.CurrentMemory.Value * 1024
	case 'b':
		memSizeMb = domain.CurrentMemory.Value / 1024 / 1024
		if domain.CurrentMemory.Value%(1024*1024) > 0 {
			memSizeMb += 1
		}
	default:
		return nil, fmt.Errorf("Unknown memory unit %s", domain.CurrentMemory.Unit)
	}

	disksConfig, err := m.LibvirtDomainDiskToDiskConfig(domain.Devices.Disks)
	if err != nil {
		return nil, err
	}

	nicsConfig, err := m.LibvirtDomainInterfaceToNicConfig(domain.Devices.Interfaces)
	if err != nil {
		return nil, err
	}

	return &compute.SImportGuestDesc{
		Id:        domain.UUID,
		Name:      domain.Name,
		Cpu:       domain.VCPU.Value,
		MemSizeMb: int(memSizeMb),
		Disks:     disksConfig,
		Nics:      nicsConfig,
	}, nil
}

func (m *SGuestManager) LibvirtDomainDiskToDiskConfig(
	domainDisks []libvirtxml.DomainDisk) ([]compute.SImportDisk, error) {
	var diskConfigs = []compute.SImportDisk{}
	for _, disk := range domainDisks {
		if disk.Device != "disk" {
			continue
		}
		if disk.Source == nil || disk.Source.File == nil {
			return nil, fmt.Errorf("Domain disk missing source file ?")
		}
		if disk.Target == nil {
			return nil, fmt.Errorf("Domain disk missing target config")
		}

		// XXX: Ignore backing file
		var diskConfig = compute.SImportDisk{
			AccessPath: strings.Trim(disk.Source.File.File, "\""),
			Index:      int(disk.Source.Index),
		}
		if disk.Target.Bus != "virtio" {
			diskConfig.Driver = "scsi"
		} else {
			diskConfig.Driver = disk.Target.Bus
		}

		if img, err := qemuimg.NewQemuImage(diskConfig.AccessPath); err != nil {
			return nil, fmt.Errorf("Domain disk %s open failed: %s", diskConfig.AccessPath, err)
		} else {
			diskConfig.SizeMb = img.GetSizeMB()
			diskConfig.Format = img.Format.String()
		}
		diskConfigs = append(diskConfigs, diskConfig)
	}
	return diskConfigs, nil
}

func (m *SGuestManager) LibvirtDomainInterfaceToNicConfig(
	domainInterfaces []libvirtxml.DomainInterface) ([]compute.SImportNic, error) {
	var nicConfigs = []compute.SImportNic{}
	for idx, nic := range domainInterfaces {
		if nic.MAC == nil {
			return nil, fmt.Errorf("Domain interface %#v missing mac address", nic)
		}
		nicConfigs = append(nicConfigs, compute.SImportNic{
			Index:  idx,
			Mac:    nic.MAC.Address,
			Driver: "virtio", //default driver
		})
	}
	return nicConfigs, nil
}