Cloudpods/backend/pkg/hostman/guestman/runtime.go

// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package guestman

import (
	"context"
	"fmt"
	"io/ioutil"
	"path"

	"yunion.io/x/jsonutils"
	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"

	computeapi "yunion.io/x/onecloud/pkg/apis/compute"
	"yunion.io/x/onecloud/pkg/hostman/guestman/desc"
	deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
	"yunion.io/x/onecloud/pkg/hostman/options"
	"yunion.io/x/onecloud/pkg/hostman/storageman"
	"yunion.io/x/onecloud/pkg/mcclient"
	"yunion.io/x/onecloud/pkg/util/cgrouputils/cpuset"
	"yunion.io/x/onecloud/pkg/util/fileutils2"
	"yunion.io/x/onecloud/pkg/util/netutils2"
	"yunion.io/x/onecloud/pkg/util/procutils"
)

type GuestRuntimeInstance interface {
	GetHypervisor() string
	GetName() string
	GetInitialId() string
	GetId() string
	IsValid() bool
	HomeDir() string
	GetDesc() *desc.SGuestDesc
	SetDesc(guestDesc *desc.SGuestDesc)
	GetSourceDesc() *desc.SGuestDesc
	SetSourceDesc(guestDesc *desc.SGuestDesc)
	GetDescFilePath() string
	NicTrafficRecordPath() string
	DeployFs(ctx context.Context, userCred mcclient.TokenCredential, deployInfo *deployapi.DeployInfo) (jsonutils.JSONObject, error)
	GetSourceDescFilePath() string
	IsRunning() bool
	IsStopped() bool
	IsSuspend() bool
	IsLoaded() bool
	GetNicDescMatch(mac, ip, port, bridge string) *desc.SGuestNetwork
	GetIpv6NicMacs(bridge string) []string
	CleanGuest(ctx context.Context, params interface{}) (jsonutils.JSONObject, error)
	CleanDirtyGuest(ctx context.Context) error
	ImportServer(pendingDelete bool)
	ExitCleanup(bool)

	HandleGuestStatus(ctx context.Context, resp *computeapi.HostUploadGuestStatusInput, isBatch bool) *computeapi.HostUploadGuestStatusInput
	GetUploadStatus(ctx context.Context, reason string) (*computeapi.HostUploadGuestStatusInput, error)
	PostUploadStatus(resp *computeapi.HostUploadGuestStatusInput, reason string)
	HandleGuestStart(ctx context.Context, userCred mcclient.TokenCredential, body jsonutils.JSONObject) (jsonutils.JSONObject, error)

	HandleStop(ctx context.Context, timeout int64) error

	LoadDesc() error
	PostLoad(m *SGuestManager) error
	SyncConfig(ctx context.Context, guestDesc *desc.SGuestDesc, fwOnly, setUefiBootOrder bool) (jsonutils.JSONObject, error)
	DoSnapshot(ctx context.Context, params *SDiskSnapshot) (jsonutils.JSONObject, error)
	DeleteSnapshot(ctx context.Context, params *SDeleteDiskSnapshot) (jsonutils.JSONObject, error)
	OnlineResizeDisk(ctx context.Context, disk storageman.IDisk, sizeMB int64)

	GetDependsImageIds(storageType string) []string

	SetNicDown(mac string) error
	SetNicUp(nic *desc.SGuestNetwork) error
}

type sBaseGuestInstance struct {
	Id      string
	manager *SGuestManager
	// runtime description, generate from source desc
	Desc *desc.SGuestDesc
	// source description, input from region
	SourceDesc *desc.SGuestDesc
	Hypervisor string
}

func newBaseGuestInstance(id string, manager *SGuestManager, hypervisor string) *sBaseGuestInstance {
	return &sBaseGuestInstance{
		Id:         id,
		manager:    manager,
		Hypervisor: hypervisor,
	}
}

func (s *sBaseGuestInstance) GetHypervisor() string {
	return s.Hypervisor
}

func (s *sBaseGuestInstance) IsValid() bool {
	return s.Desc != nil && s.Desc.Uuid != ""
}

func (s *sBaseGuestInstance) GetInitialId() string {
	return s.Id
}

func (s *sBaseGuestInstance) GetId() string {
	return s.Desc.Uuid
}

func (s *sBaseGuestInstance) GetName() string {
	return fmt.Sprintf("%s(%s)", s.Desc.Name, s.Desc.Uuid)
}

func (b *sBaseGuestInstance) HomeDir() string {
	return path.Join(b.manager.ServersPath, b.Id)
}

func (s *sBaseGuestInstance) GetDescFilePath() string {
	return path.Join(s.HomeDir(), "desc")
}

func (s *sBaseGuestInstance) GetDesc() *desc.SGuestDesc {
	return s.Desc
}

func (s *sBaseGuestInstance) SetDesc(guestDesc *desc.SGuestDesc) {
	s.Desc = guestDesc
}

func (s *sBaseGuestInstance) GetSourceDesc() *desc.SGuestDesc {
	return s.SourceDesc
}

func (s *sBaseGuestInstance) SetSourceDesc(guestDesc *desc.SGuestDesc) {
	s.SourceDesc = guestDesc
}

func (s *sBaseGuestInstance) GetSourceDescFilePath() string {
	return path.Join(s.HomeDir(), "source-desc")
}

func (s *sBaseGuestInstance) NicTrafficRecordPath() string {
	return path.Join(s.HomeDir(), "nic_traffic.json")
}

func (s *sBaseGuestInstance) IsLoaded() bool {
	return s.Desc != nil
}

func (s *sBaseGuestInstance) IsDaemon() bool {
	return s.Desc.IsDaemon
}

func (s *sBaseGuestInstance) GetNicDescMatch(mac, ip, port, bridge string) *desc.SGuestNetwork {
	nics := s.Desc.Nics
	for _, nic := range nics {
		if bridge == "" && nic.Bridge != "" && nic.Bridge == options.HostOptions.OvnIntegrationBridge {
			continue
		}
		if (len(mac) == 0 || netutils2.MacEqual(nic.Mac, mac)) &&
			(len(ip) == 0 || nic.Ip == ip) &&
			(len(port) == 0 || nic.Ifname == port) &&
			(len(bridge) == 0 || nic.Bridge == bridge) {
			return nic
		}
	}
	return nil
}

func (s *sBaseGuestInstance) GetIpv6NicMacs(bridge string) []string {
	macs := []string{}
	for _, nic := range s.Desc.Nics {
		if nic.Bridge == bridge && len(nic.Ip6) > 0 {
			macs = append(macs, nic.Mac)
		}
	}
	return macs
}

func LoadGuestCpuset(m *SGuestManager, s GuestRuntimeInstance) error {
	guestDesc := s.GetDesc()
	if s.IsRunning() {
		m.cpuSet.Lock.Lock()
		defer m.cpuSet.Lock.Unlock()
		m.cpuSet.GuestIds[s.GetId()] = struct{}{}

		for _, vcpuPin := range guestDesc.VcpuPin {
			pcpuSet, err := cpuset.Parse(vcpuPin.Pcpus)
			if err != nil {
				log.Errorf("failed parse %s pcpus: %s", s.GetName(), vcpuPin.Pcpus)
				continue
			}
			vcpuSet, err := cpuset.Parse(vcpuPin.Vcpus)
			if err != nil {
				log.Errorf("failed parse %s vcpus: %s", s.GetName(), vcpuPin.Vcpus)
				continue
			}
			m.cpuSet.LoadCpus(pcpuSet.ToSlice(), vcpuSet.Size())
		}

		for _, numaCpuset := range guestDesc.CpuNumaPin {
			pcpus := make([]int, 0)
			vcpus := make([]int, 0)
			for i := range numaCpuset.VcpuPin {
				pcpus = append(pcpus, numaCpuset.VcpuPin[i].Pcpu)
				if numaCpuset.VcpuPin[i].Vcpu >= 0 {
					vcpus = append(vcpus, numaCpuset.VcpuPin[i].Vcpu)
				}
			}

			m.cpuSet.LoadNumaCpus(numaCpuset.SizeMB, int(*numaCpuset.NodeId), pcpus, len(vcpus))
		}
	}
	return nil
}

func ReleaseCpuNumaPin(m *SGuestManager, cpuNumaPin []*desc.SCpuNumaPin) {
	if !m.hostagentNumaAllocate {
		return
	}

	for _, numaCpus := range cpuNumaPin {
		pcpus := make([]int, 0)
		vcpus := make([]int, 0)
		for i := range numaCpus.VcpuPin {
			pcpus = append(pcpus, numaCpus.VcpuPin[i].Pcpu)
			if numaCpus.VcpuPin[i].Vcpu >= 0 {
				vcpus = append(vcpus, numaCpus.VcpuPin[i].Vcpu)
			}
		}
		m.cpuSet.ReleaseNumaCpus(numaCpus.SizeMB, int(*numaCpus.NodeId), pcpus, len(vcpus))
	}
}

func ReleaseGuestCpuset(m *SGuestManager, s GuestRuntimeInstance) {
	m.cpuSet.Lock.Lock()
	defer m.cpuSet.Lock.Unlock()
	delete(m.cpuSet.GuestIds, s.GetId())

	guestDesc := s.GetDesc()
	ReleaseCpuNumaPin(m, guestDesc.CpuNumaPin)

	for _, vcpuPin := range guestDesc.VcpuPin {
		pcpuSet, err := cpuset.Parse(vcpuPin.Pcpus)
		if err != nil {
			log.Errorf("failed parse %s pcpus: %s", s.GetName(), vcpuPin.Pcpus)
			continue
		}
		vcpuSet, err := cpuset.Parse(vcpuPin.Vcpus)
		if err != nil {
			log.Errorf("failed parse %s vcpus: %s", s.GetName(), vcpuPin.Vcpus)
			continue
		}
		m.cpuSet.ReleaseCpus(pcpuSet.ToSlice(), vcpuSet.Size())
	}
	guestDesc.VcpuPin = nil
	guestDesc.CpuNumaPin = nil
	SaveLiveDesc(s, guestDesc)
}

type GuestRuntimeManager struct {
}

func NewGuestRuntimeManager() *GuestRuntimeManager {
	return new(GuestRuntimeManager)
}

func (f *GuestRuntimeManager) NewRuntimeInstance(id string, manager *SGuestManager, hypervisor string) GuestRuntimeInstance {
	switch hypervisor {
	case computeapi.HYPERVISOR_KVM, "":
		return NewKVMGuestInstance(id, manager)
	case computeapi.HYPERVISOR_POD:
		return newPodGuestInstance(id, manager)
	}
	log.Fatalf("Invalid hypervisor for runtime: %q", hypervisor)
	return nil
}

func PrepareDir(s GuestRuntimeInstance) error {
	output, err := procutils.NewCommand("mkdir", "-p", s.HomeDir()).Output()
	if err != nil {
		return errors.Wrapf(err, "mkdir %s failed: %s", s.HomeDir(), output)
	}
	return nil
}

func (f *GuestRuntimeManager) CreateFromDesc(s GuestRuntimeInstance, desc *desc.SGuestDesc) error {
	if err := PrepareDir(s); err != nil {
		return errors.Errorf("Failed to create server dir %s", desc.Uuid)
	}
	return SaveDesc(s, desc)
}

func (s *sBaseGuestInstance) GetVpcNIC() *desc.SGuestNetwork {
	for _, nic := range s.Desc.Nics {
		if nic.Vpc.Provider == computeapi.VPC_PROVIDER_OVN {
			if nic.Ip != "" {
				return nic
			}
		}
	}
	return nil
}

func (s *sBaseGuestInstance) UpdateLiveDesc(guestDesc *desc.SGuestDesc) {
	// update guest live desc, don't be here update cpu and mem
	// cpu and memory should update from SGuestHotplugCpuMemTask
	s.Desc.SGuestControlDesc = guestDesc.SGuestControlDesc
	s.Desc.SGuestProjectDesc = guestDesc.SGuestProjectDesc
	s.Desc.SGuestRegionDesc = guestDesc.SGuestRegionDesc
	s.Desc.SGuestMetaDesc = guestDesc.SGuestMetaDesc
}

func LoadDesc(s GuestRuntimeInstance) error {
	descPath := s.GetDescFilePath()
	descStr, err := ioutil.ReadFile(descPath)
	if err != nil {
		return errors.Wrap(err, "read desc")
	}

	var (
		srcDescStr  []byte
		srcDescPath = s.GetSourceDescFilePath()
	)
	if !fileutils2.Exists(srcDescPath) {
		err = fileutils2.FilePutContents(srcDescPath, string(descStr), false)
		if err != nil {
			return errors.Wrap(err, "save source desc")
		}
		srcDescStr = descStr
	} else {
		srcDescStr, err = ioutil.ReadFile(srcDescPath)
		if err != nil {
			return errors.Wrap(err, "read source desc")
		}
	}

	// parse source desc
	srcGuestDesc := new(desc.SGuestDesc)
	jsonSrcDesc, err := jsonutils.Parse(srcDescStr)
	if err != nil {
		return errors.Wrap(err, "json parse source desc")
	}
	err = jsonSrcDesc.Unmarshal(srcGuestDesc)
	if err != nil {
		return errors.Wrap(err, "unmarshal source desc")
	}
	s.SetSourceDesc(srcGuestDesc)

	// parse desc
	guestDesc := new(desc.SGuestDesc)
	jsonDesc, err := jsonutils.Parse(descStr)
	if err != nil {
		return errors.Wrap(err, "json parse desc")
	}
	err = jsonDesc.Unmarshal(guestDesc)
	if err != nil {
		return errors.Wrap(err, "unmarshal desc")
	}
	s.SetDesc(guestDesc)
	return nil
}

func SaveDesc(s GuestRuntimeInstance, guestDesc *desc.SGuestDesc) error {
	s.SetSourceDesc(guestDesc)
	// fill in ovn vpc nic bridge field
	for _, nic := range s.GetSourceDesc().Nics {
		if nic.Bridge == "" {
			nic.Bridge = getNicBridge(nic)
		}
	}

	if err := fileutils2.FilePutContents(
		s.GetSourceDescFilePath(), jsonutils.Marshal(s.GetSourceDesc()).String(), false,
	); err != nil {
		log.Errorf("save source desc failed %s", err)
		return errors.Wrap(err, "source save desc")
	}

	if !s.IsRunning() { // if guest not running, sync live desc
		liveDesc := new(desc.SGuestDesc)
		if err := jsonutils.Marshal(s.GetSourceDesc()).Unmarshal(liveDesc); err != nil {
			return errors.Wrap(err, "unmarshal live desc")
		}
		return SaveLiveDesc(s, liveDesc)
	}
	return nil
}

func SaveLiveDesc(s GuestRuntimeInstance, guestDesc *desc.SGuestDesc) error {
	s.SetDesc(guestDesc)

	defaultGwCnt := 0
	defNics := netutils2.SNicInfoList{}
	// fill in ovn vpc nic bridge field
	for _, nic := range s.GetDesc().Nics {
		if nic.Bridge == "" {
			nic.Bridge = getNicBridge(nic)
		}
		if nic.IsDefault {
			defaultGwCnt++
		}
		defNics = defNics.Add(nic.Mac, nic.Ip, nic.Gateway, nic.Ip6, nic.Gateway6, nic.IsDefault)
	}

	// there should 1 and only 1 default gateway
	if defaultGwCnt != 1 {
		// fix is_default
		_, defIndex := defNics.FindDefaultNicMac()
		for i := range s.GetDesc().Nics {
			if i == defIndex {
				s.GetDesc().Nics[i].IsDefault = true
			} else {
				s.GetDesc().Nics[i].IsDefault = false
			}
		}
	}

	if err := fileutils2.FilePutContents(
		s.GetDescFilePath(), jsonutils.Marshal(s.GetDesc()).String(), false,
	); err != nil {
		log.Errorf("save desc failed %s", err)
		return errors.Wrap(err, "save desc")
	}
	return nil
}

func GetPowerStates(s GuestRuntimeInstance) string {
	if s.IsRunning() {
		return computeapi.VM_POWER_STATES_ON
	} else {
		return computeapi.VM_POWER_STATES_OFF
	}
}

func DeleteHomeDir(s GuestRuntimeInstance) error {
	output, err := procutils.NewCommand("rm", "-rf", s.HomeDir()).Output()
	if err != nil {
		return errors.Wrapf(err, "rm %s failed: %s", s.HomeDir(), output)
	}
	return nil
}