Cloudpods/backend/pkg/baremetal/manager.go

// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package baremetal

import (
	"context"
	"fmt"
	"io/fs"
	"io/ioutil"
	"net"
	"net/http"
	"net/url"
	"os"
	"path/filepath"
	"reflect"
	"strconv"
	"strings"
	"sync"
	"time"

	"yunion.io/x/cloudmux/pkg/apis/compute"
	"yunion.io/x/jsonutils"
	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"
	"yunion.io/x/pkg/util/httputils"
	"yunion.io/x/pkg/util/netutils"
	"yunion.io/x/pkg/util/regutils"
	"yunion.io/x/pkg/util/seclib"
	"yunion.io/x/pkg/util/sets"
	"yunion.io/x/pkg/util/workqueue"
	"yunion.io/x/pkg/utils"

	"yunion.io/x/onecloud/pkg/apis"
	api "yunion.io/x/onecloud/pkg/apis/compute"
	baremetalapi "yunion.io/x/onecloud/pkg/apis/compute/baremetal"
	apiidenty "yunion.io/x/onecloud/pkg/apis/identity"
	o "yunion.io/x/onecloud/pkg/baremetal/options"
	"yunion.io/x/onecloud/pkg/baremetal/profiles"
	"yunion.io/x/onecloud/pkg/baremetal/pxe"
	baremetalstatus "yunion.io/x/onecloud/pkg/baremetal/status"
	"yunion.io/x/onecloud/pkg/baremetal/tasks"
	baremetaltypes "yunion.io/x/onecloud/pkg/baremetal/types"
	"yunion.io/x/onecloud/pkg/baremetal/utils/detect_storages"
	"yunion.io/x/onecloud/pkg/baremetal/utils/disktool"
	"yunion.io/x/onecloud/pkg/baremetal/utils/grub"
	"yunion.io/x/onecloud/pkg/baremetal/utils/ipmitool"
	raid2 "yunion.io/x/onecloud/pkg/baremetal/utils/raid"
	raiddrivers "yunion.io/x/onecloud/pkg/baremetal/utils/raid/drivers"
	"yunion.io/x/onecloud/pkg/baremetal/utils/raid/mdadm"
	"yunion.io/x/onecloud/pkg/baremetal/utils/uefi"
	"yunion.io/x/onecloud/pkg/cloudcommon/types"
	"yunion.io/x/onecloud/pkg/compute/baremetal"
	"yunion.io/x/onecloud/pkg/hostman/guestfs"
	"yunion.io/x/onecloud/pkg/hostman/guestfs/sshpart"
	deployapi "yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
	"yunion.io/x/onecloud/pkg/httperrors"
	"yunion.io/x/onecloud/pkg/mcclient"
	"yunion.io/x/onecloud/pkg/mcclient/auth"
	modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
	"yunion.io/x/onecloud/pkg/util/dhcp"
	"yunion.io/x/onecloud/pkg/util/fileutils2"
	"yunion.io/x/onecloud/pkg/util/influxdb"
	"yunion.io/x/onecloud/pkg/util/procutils"
	"yunion.io/x/onecloud/pkg/util/redfish"
	"yunion.io/x/onecloud/pkg/util/redfish/bmconsole"
	"yunion.io/x/onecloud/pkg/util/ssh"
	"yunion.io/x/onecloud/pkg/util/sysutils"
)

type SBaremetalManager struct {
	Agent      *SBaremetalAgent
	configPath string
	baremetals *sBaremetalMap
}

func NewBaremetalManager(agent *SBaremetalAgent) (*SBaremetalManager, error) {
	bmPaths := o.Options.BaremetalsPath
	err := os.MkdirAll(bmPaths, 0755)
	if err != nil {
		return nil, err
	}
	return &SBaremetalManager{
		Agent:      agent,
		configPath: bmPaths,
		baremetals: newBaremetalMap(),
	}, nil
}

func (m *SBaremetalManager) killAllIPMITool() {
	procutils.NewCommand("killall", "-9", "ipmitool").Run()
}

func (m *SBaremetalManager) GetClientSession() *mcclient.ClientSession {
	return m.Agent.GetAdminSession()
}

func (m *SBaremetalManager) GetPublicClientSession() *mcclient.ClientSession {
	return m.Agent.GetPublicAdminSession()
}

func (m *SBaremetalManager) GetZoneId() string {
	return m.Agent.Zone.Id
}

func (m *SBaremetalManager) GetZoneName() string {
	return m.Agent.Zone.Name
}

func (m *SBaremetalManager) loadConfigs() ([]fs.DirEntry, error) {
	m.killAllIPMITool()
	files, err := os.ReadDir(m.configPath)
	if err != nil {
		return nil, err
	}
	return files, nil
}

func (m *SBaremetalManager) initBaremetals(ctx context.Context, files []fs.DirEntry) error {
	bmIds := make([]string, 0)
	for _, file := range files {
		if file.IsDir() && regutils.MatchUUID(file.Name()) {
			bmIds = append(bmIds, file.Name())
		}
	}

	errsChannel := make(chan error, len(bmIds))
	initBaremetal := func(i int) {
		bmId := bmIds[i]
		err := m.InitBaremetal(ctx, bmId, true)
		if err != nil {
			errsChannel <- err
			return
		}
	}
	workqueue.Parallelize(4, len(bmIds), initBaremetal)
	errs := make([]error, 0)
	if len(errsChannel) > 0 {
		length := len(errsChannel)
		for ; length > 0; length-- {
			errs = append(errs, <-errsChannel)
		}
	}
	return errors.NewAggregate(errs)
}

func (m *SBaremetalManager) InitBaremetal(ctx context.Context, bmId string, update bool) error {
	session := m.GetClientSession()
	var err error
	var desc jsonutils.JSONObject
	if update {
		desc, err = m.updateBaremetal(session, bmId)
	} else {
		desc, err = m.fetchBaremetal(session, bmId)
	}
	if err != nil {
		return err
	}
	isBaremetal, _ := desc.Bool("is_baremetal")
	if !isBaremetal {
		return errors.Error("not a baremetal???")
	}
	bmInstance, err := m.AddBaremetal(ctx, desc)
	if err != nil {
		return err
	}
	if update {
		bmObj := bmInstance.(*SBaremetalInstance)
		if !sets.NewString(INIT, PREPARE, UNKNOWN).Has(bmObj.GetStatus()) {
			if !bmObj.IsHypervisorHost() {
				bmObj.SyncStatusBackground()
			}
		}
	}
	return nil
}

func (m *SBaremetalManager) CleanBaremetal(bmId string) {
	bm := m.baremetals.Pop(bmId)
	if bm != nil {
		bm.Stop()
	}
	bm.clearBootIsoImage()
	path := bm.GetDir()
	procutils.NewCommand("rm", "-fr", path).Run()
}

func (m *SBaremetalManager) updateBaremetal(session *mcclient.ClientSession, bmId string) (jsonutils.JSONObject, error) {
	params := jsonutils.NewDict()
	params.Add(jsonutils.JSONTrue, "is_baremetal")
	params.Add(jsonutils.JSONTrue, "not_sync_config")
	obj, err := modules.Hosts.Put(session, bmId, params)
	if err != nil {
		return nil, err
	}
	log.Infof("Baremetal %s update success", bmId)
	return obj, nil
}

func (m *SBaremetalManager) fetchBaremetal(session *mcclient.ClientSession, bmId string) (jsonutils.JSONObject, error) {
	obj, err := modules.Hosts.Get(session, bmId, nil)
	if err != nil {
		return nil, err
	}
	log.Infof("Baremetal %s update success", bmId)
	return obj, nil
}

func (m *SBaremetalManager) AddBaremetal(ctx context.Context, desc jsonutils.JSONObject) (pxe.IBaremetalInstance, error) {
	id, err := desc.GetString("id")
	if err != nil {
		return nil, fmt.Errorf("Not found baremetal id in desc %s", desc)
	}
	if instance, ok := m.baremetals.Get(id); ok {
		return instance, instance.SaveDesc(ctx, desc)
	}
	bm, err := newBaremetalInstance(ctx, m, desc)
	if err != nil {
		return nil, err
	}
	m.baremetals.Add(bm)
	return bm, nil
}

func (m *SBaremetalManager) GetBaremetals() []*SBaremetalInstance {
	objs := make([]*SBaremetalInstance, 0)
	getter := func(key, val interface{}) bool {
		objs = append(objs, val.(*SBaremetalInstance))
		return true
	}
	m.baremetals.Range(getter)
	return objs
}

func (m *SBaremetalManager) GetBaremetalById(bmId string) *SBaremetalInstance {
	obj, _ := m.baremetals.Get(bmId)
	return obj
}

func (m *SBaremetalManager) GetBaremetalByMac(mac net.HardwareAddr) pxe.IBaremetalInstance {
	var obj pxe.IBaremetalInstance
	getter := func(key, val interface{}) bool {
		instance := val.(*SBaremetalInstance)
		if instance.GetNicByMac(mac) != nil {
			obj = instance
			// stop the iteration
			return false
		}
		// not found, continue iteration
		return true
	}
	m.baremetals.Range(getter)
	return obj
}

type BmRegisterInput struct {
	// context with timeout
	Ctx context.Context

	R *http.Request
	W http.ResponseWriter

	// For notify web server close this connection
	C chan struct{}

	// remote server ssh info
	SshPort   int
	SshPasswd string
	Hostname  string
	RemoteIp  string

	// ipmi info
	Username string
	Password string
	IpAddr   string
}

func (i *BmRegisterInput) responseSucc(bmId string) {
	fmt.Fprintf(i.W, "%s", bmId)
	close(i.C)
}

func (i *BmRegisterInput) responseErr(ctx context.Context, err error) {
	httperrors.GeneralServerError(ctx, i.W, err)
	close(i.C)
}

func (i *BmRegisterInput) isTimeout() bool {
	select {
	case <-i.Ctx.Done():
		return true
	default:
		return false
	}
}

// delay task
func (m *SBaremetalManager) RegisterBaremetal(ctx context.Context, userCred mcclient.TokenCredential, input *BmRegisterInput) {
	adminWire, err := m.checkNetworkFromIp(input.RemoteIp)
	if input.isTimeout() {
		return
	} else if err != nil {
		input.responseErr(ctx, httperrors.NewBadRequestError("Verify network failed: %s", err))
		return
	}

	sshCli, err := m.checkSshInfo(input)
	if input.isTimeout() {
		return
	} else if err != nil {
		input.responseErr(ctx, httperrors.NewBadRequestError("SSH verify failed: %s", err))
		return
	}

	isIpv6Addr := false
	if strings.Contains(input.RemoteIp, ":") {
		isIpv6Addr = true
	}
	input.IpAddr, err = m.fetchIpmiIp(sshCli, isIpv6Addr)
	log.Infof("find ipmi addr %s", input.IpAddr)

	if input.isTimeout() {
		return
	} else if err != nil {
		input.responseErr(ctx, httperrors.NewBadRequestError("Fetch ipmi address failed: %s", err))
		return
	}
	log.Infof("Find ipmi address %s", input.IpAddr)

	ipmiWire, err := m.checkNetworkFromIp(input.IpAddr)
	if input.isTimeout() {
		return
	} else if err != nil {
		input.responseErr(ctx, httperrors.NewBadRequestError("Verify network failed: %s", err))
		return
	}

	ipmiLanChannel, ipmiMac, err := m.checkIpmiInfo(ctx, input.Username, input.Password, input.IpAddr)
	if input.isTimeout() {
		return
	} else if err != nil {
		input.responseErr(ctx, httperrors.NewBadRequestError("IPMI login info not correct: %s", err))
		return
	}

	err, registered := m.verifyMacAddr(sshCli)
	if input.isTimeout() {
		return
	} else if err != nil {
		input.responseErr(ctx, httperrors.NewBadRequestError("Verify mac address failed: %s", err))
		return
	}

	registerTask := tasks.NewBaremetalRegisterTask(
		userCred, m, sshCli, input.Hostname, input.RemoteIp,
		input.Username, input.Password, input.IpAddr,
		ipmiMac, ipmiLanChannel, adminWire, ipmiWire,
	)
	var bmId string
	if !registered {
		bmId, err = registerTask.CreateBaremetal(ctx)
	} else {
		bmId, err = registerTask.UpdateBaremetal(ctx)
	}
	if err != nil {
		input.responseErr(ctx, httperrors.NewInternalServerError("%v", err))
		return
	}

	input.responseSucc(bmId)
	registerTask.DoPrepare(ctx, sshCli, registered)
}

func (m *SBaremetalManager) fetchIpmiIp(sshCli *ssh.Client, isIpv6Addr bool) (string, error) {
	if !isIpv6Addr {
		res, err := sshCli.RawRun(`/usr/bin/ipmitool lan print | grep "IP Address  "`)
		if err != nil {
			return "", err
		}
		if len(res) == 1 {
			segs := strings.Fields(res[0])
			if len(segs) == 4 {
				return strings.TrimSpace(segs[3]), nil
			}
		}
	} else {
		res, err := sshCli.Run(`/usr/bin/ipmitool lan6 print`)
		if err != nil {
			return "", err
		}
		for i, line := range res {
			if strings.HasPrefix(line, "IPv6 Static Address") || strings.HasPrefix(line, "IPv6 Dynamic Address") {
				if len(res)-i > 3 {
					enabledSegs := strings.Fields(res[i+1])
					log.Infof("enabled segs %#v", enabledSegs)
					if len(enabledSegs) != 2 || enabledSegs[0] != "Enabled:" || enabledSegs[1] != "yes" {
						continue
					}
					statusSegs := strings.Fields(res[i+3])
					log.Infof("status segs %#v", statusSegs)
					if len(enabledSegs) != 2 || statusSegs[0] != "Status:" || statusSegs[1] != "active" {
						continue
					}
					addrSegs := strings.Fields(res[i+2])
					if len(addrSegs) != 2 || addrSegs[0] != "Address:" {
						continue
					}
					ipv6Addr := strings.Split(addrSegs[1], "/")

					return ipv6Addr[0], nil
				}
			}
		}
	}

	return "", fmt.Errorf("Failed to find ipmi ip address")
}

func (m *SBaremetalManager) checkNetworkFromIp(ip string) (string, error) {
	params := jsonutils.NewDict()
	params.Set("ip", jsonutils.NewString(ip))
	params.Set("scope", jsonutils.NewString("system"))
	params.Set("is_classic", jsonutils.JSONTrue)
	params.Set("provider", jsonutils.NewString(api.CLOUD_PROVIDER_ONECLOUD))
	params.Set("limit", jsonutils.NewInt(0))
	// use default vpc
	params.Set("vpc", jsonutils.NewString(api.DEFAULT_VPC_ID))

	res, err := modules.Networks.List(m.GetClientSession(), params)
	if err != nil {
		return "", fmt.Errorf("Fetch network by ip %s failed: %s", ip, err)
	}
	if len(res.Data) != 1 {
		return "", fmt.Errorf("Can't find network from ip %s", ip)
	}
	return res.Data[0].GetString("wire_id")
}

func (m *SBaremetalManager) verifyMacAddr(sshCli *ssh.Client) (error, bool) {
	output, err := sshCli.Run("/lib/mos/lsnic")
	if err != nil {
		return err, false
	}
	nicinfo := sysutils.ParseNicInfo(output)
	if len(nicinfo) == 0 {
		return fmt.Errorf("Can't get nic info"), false
	}

	var registered bool
	params := jsonutils.NewDict()
	for _, nic := range nicinfo {
		if len(nic.Mac) > 0 {
			params.Set("any_mac", jsonutils.NewString(nic.Mac.String()))
			params.Set("scope", jsonutils.NewString("system"))
			res, err := modules.Hosts.List(m.GetClientSession(), params)
			if err != nil {
				return fmt.Errorf("Get hosts info failed: %s", err), false
			}
			if len(res.Data) > 0 {
				registered = true
			}
		}
	}
	return nil, registered
}

func (m *SBaremetalManager) checkSshInfo(input *BmRegisterInput) (*ssh.Client, error) {
	sshCLi, err := ssh.NewClient(input.RemoteIp, input.SshPort, "root", input.SshPasswd, "")
	if err != nil {
		return nil, fmt.Errorf("Ssh connect failed: %s", err)
	}

	var RETRY, MAX_TRIES = 0, 3
	for RETRY = 0; RETRY < MAX_TRIES; RETRY++ {
		_, err := sshCLi.Run("/bin/ls")
		if err != nil {
			log.Errorf("Exec remote command failed: %s", err)
			time.Sleep(time.Second * 1)
		} else {
			break
		}
	}
	if RETRY >= MAX_TRIES {
		return nil, fmt.Errorf("SSH login info not correct??")
	}
	return sshCLi, nil
}

func (m *SBaremetalManager) checkIpmiInfo(ctx context.Context, username, password, ipAddr string) (uint8, net.HardwareAddr, error) {
	lanPlusTool := ipmitool.NewLanPlusIPMI(ipAddr, username, password)
	sysInfo, err := ipmitool.GetSysInfo(lanPlusTool)
	if err != nil {
		return 0, nil, errors.Wrap(err, "GetSysInfo")
	}
	profile, err := profiles.GetProfile(ctx, sysInfo)
	if err != nil {
		return 0, nil, errors.Wrap(err, "GetProfile")
	}

	for _, lanChannel := range profile.LanChannels {
		config, err := ipmitool.GetLanConfig(lanPlusTool, lanChannel)
		if err != nil {
			log.Errorf("GetLanConfig failed %s", err)
			continue
		}
		if len(config.Mac) == 0 {
			continue
		}
		return lanChannel, config.Mac, nil
	}
	return 0, nil, fmt.Errorf("Ipmi can't fetch lan config")
}

func (m *SBaremetalManager) Stop() {
	for _, bm := range m.GetBaremetals() {
		bm.Stop()
	}
}

type sBaremetalMap struct {
	*sync.Map
}

func newBaremetalMap() *sBaremetalMap {
	return &sBaremetalMap{
		Map: new(sync.Map),
	}
}

func (m *sBaremetalMap) Add(bm *SBaremetalInstance) {
	m.Store(bm.GetId(), bm)
}

func (m *sBaremetalMap) Get(id string) (*SBaremetalInstance, bool) {
	obj, ok := m.Load(id)
	if !ok {
		return nil, false
	}
	return obj.(*SBaremetalInstance), true
}

func (m *sBaremetalMap) Delete(id string) {
	m.Map.Delete(id)
}

func (m *sBaremetalMap) Pop(id string) *SBaremetalInstance {
	obj, exist := m.Get(id)
	if exist {
		m.Delete(id)
	}
	return obj
}

type SBaremetalInstance struct {
	manager    *SBaremetalManager
	desc       *jsonutils.JSONDict
	descLock   *sync.Mutex
	taskQueue  *tasks.TaskQueue
	server     baremetaltypes.IBaremetalServer
	serverLock *sync.Mutex

	profile *baremetalapi.BaremetalProfileSpec

	cronJobs []IBaremetalCronJob
}

func newBaremetalInstance(ctx context.Context, man *SBaremetalManager, desc jsonutils.JSONObject) (*SBaremetalInstance, error) {
	bm := &SBaremetalInstance{
		manager:    man,
		desc:       desc.(*jsonutils.JSONDict),
		descLock:   new(sync.Mutex),
		taskQueue:  tasks.NewTaskQueue(),
		serverLock: new(sync.Mutex),
	}
	bm.cronJobs = []IBaremetalCronJob{
		NewLogFetchJob(bm, time.Duration(o.Options.LogFetchIntervalSeconds)*time.Second),
		NewSendMetricsJob(bm, time.Duration(o.Options.SendMetricsIntervalSeconds)*time.Second),
		NewStatusProbeJob(bm, time.Duration(o.Options.StatusProbeIntervalSeconds)*time.Second),
	}
	err := os.MkdirAll(bm.GetDir(), 0755)
	if err != nil {
		return nil, err
	}
	err = bm.SaveDesc(ctx, desc)
	if err != nil {
		return nil, err
	}
	bm.loadServer()
	return bm, nil
}

func (b *SBaremetalInstance) IsHypervisorHost() bool {
	hostType, _ := b.desc.GetString("host_type")
	return utils.IsInStringArray(hostType, []string{api.HOST_TYPE_KVM, api.HOST_TYPE_HYPERVISOR})
}

func (b *SBaremetalInstance) GetDHCPServerIP() (net.IP, error) {
	return b.manager.Agent.GetDHCPServerIP()
}

func (b *SBaremetalInstance) GetClientSession() *mcclient.ClientSession {
	return b.manager.GetClientSession()
}

func (b *SBaremetalInstance) GetPublicClientSession() *mcclient.ClientSession {
	return b.manager.GetPublicClientSession()
}

func (b *SBaremetalInstance) Keyword() string {
	return "host"
}

func (b *SBaremetalInstance) GetId() string {
	id, err := b.desc.GetString("id")
	if err != nil {
		log.Errorf("Get id from desc %s error: %v", b.desc.String(), err)
	}
	return id
}

func (b *SBaremetalInstance) GetName() string {
	id, err := b.desc.GetString("name")
	if err != nil {
		log.Fatalf("Get name from desc %s error: %v", b.desc.String(), err)
	}
	return id
}

func (b *SBaremetalInstance) Stop() {
	// TODO:
}

func (b *SBaremetalInstance) GetDir() string {
	return filepath.Join(b.manager.configPath, b.GetId())
}

func (b *SBaremetalInstance) GetDescFilePath() string {
	return filepath.Join(b.GetDir(), "desc")
}

func (b *SBaremetalInstance) GetServerDescFilePath() string {
	return filepath.Join(b.GetDir(), "server")
}

func (b *SBaremetalInstance) GetSSHConfigFilePath() string {
	return filepath.Join(b.GetDir(), "ssh")
}

func (b *SBaremetalInstance) GetStatus() string {
	status, err := b.desc.GetString("status")
	if err != nil {
		log.Fatalf("Get status from desc error: %v", err)
	}
	return status
}

func (b *SBaremetalInstance) AutoSaveDesc(ctx context.Context) error {
	return b.SaveDesc(ctx, nil)
}

func (b *SBaremetalInstance) SaveDesc(ctx context.Context, desc jsonutils.JSONObject) error {
	b.descLock.Lock()
	defer b.descLock.Unlock()
	if desc != nil {
		if b.desc != nil && b.desc.Contains("server_id") && !desc.Contains("server_id") {
			if b.server != nil {
				desc.(*jsonutils.JSONDict).Set("server_id", jsonutils.NewString(b.server.GetId()))
			}
		}
		b.desc = desc.(*jsonutils.JSONDict)
		if b.desc.Contains("sys_info") {
			sysInfo := types.SSystemInfo{}
			err := b.desc.Unmarshal(&sysInfo, "sys_info")
			if err != nil {
				return errors.Wrap(err, "Unmarshal sys_info")
			}
			b.profile, err = profiles.GetProfile(ctx, &sysInfo)
			if err != nil {
				return errors.Wrap(err, "GetProfile")
			}
		}
	}
	return os.WriteFile(b.GetDescFilePath(), []byte(b.desc.String()), 0644)
}

func (b *SBaremetalInstance) loadServer() {
	b.serverLock.Lock()
	defer b.serverLock.Unlock()
	if !b.desc.Contains("server_id") {
		return
	}
	descPath := b.GetServerDescFilePath()
	desc, err := os.ReadFile(descPath)
	if err != nil {
		log.Errorf("Failed to read server desc %s: %v", descPath, err)
		return
	}
	descObj, err := jsonutils.Parse(desc)
	if err != nil {
		log.Errorf("Failed to parse server json string: %v", err)
		return
	}
	srv, err := newBaremetalServer(b, descObj.(*jsonutils.JSONDict))
	if err != nil {
		log.Errorf("New server error: %v", err)
		return
	}
	if bmSrvId, _ := b.desc.GetString("server_id"); srv.GetId() != bmSrvId {
		log.Errorf("Server id %q not equal baremetal %q server id %q", srv.GetId(), b.GetName(), bmSrvId)
		return
	}
	b.server = srv
}

func (b *SBaremetalInstance) SaveSSHConfig(remoteAddr string, key string) error {
	var err error
	key, err = utils.EncryptAESBase64(b.GetId(), key)
	if err != nil {
		return err
	}
	sshConf := types.SSHConfig{
		Username: "root",
		Password: key,
		RemoteIP: remoteAddr,
	}
	conf := jsonutils.Marshal(sshConf)
	err = os.WriteFile(b.GetSSHConfigFilePath(), []byte(conf.String()), 0644)
	if err != nil {
		return err
	}
	b.SyncSSHConfig(sshConf)
	b.clearBootIso()
	return err
}

func (b *SBaremetalInstance) GetSSHConfig() (*types.SSHConfig, error) {
	path := b.GetSSHConfigFilePath()
	content, err := os.ReadFile(path)
	if err != nil {
		if os.IsNotExist(err) {
			return nil, nil
		}
		return nil, err
	}
	conf := types.SSHConfig{}
	obj, err := jsonutils.Parse(content)
	if err != nil {
		return nil, err
	}
	err = obj.Unmarshal(&conf)
	if err != nil {
		return nil, err
	}
	if len(conf.RemoteIP) == 0 {
		return nil, nil
	}
	conf.Password, err = utils.DescryptAESBase64(b.GetId(), conf.Password)
	if err != nil {
		return nil, err
	}
	return &conf, nil
}

func (b *SBaremetalInstance) TestSSHConfig() bool {
	conf, err := b.GetSSHConfig()
	if err != nil {
		return false
	}
	if conf == nil {
		return false
	}
	sshCli, err := ssh.NewClient(conf.RemoteIP, 22, "root", conf.Password, "")
	if err != nil {
		return false
	}
	ret, err := sshCli.Run("whoami")
	if err != nil {
		return false
	}
	if strings.Contains(strings.Join(ret, ""), "root") {
		return true
	}
	return false
}

func (b *SBaremetalInstance) ClearSSHConfig() {
	path := b.GetSSHConfigFilePath()
	err := os.Remove(path)
	if err != nil {
		log.V(2).Warningf("Clear ssh config %s error: %v", path, err)
	}
	emptyConfig := types.SSHConfig{
		Username: "None",
		Password: "None",
		RemoteIP: "None",
	}
	err = b.SyncSSHConfig(emptyConfig)
	if err != nil {
		log.Errorf("Sync emtpy SSH config error: %v", err)
	}
}

func (b *SBaremetalInstance) SyncSSHConfig(conf types.SSHConfig) error {
	session := b.manager.GetClientSession()
	var info *api.HostLoginInfo
	if len(conf.RemoteIP) > 0 {
		var err error
		// encrypt twice
		conf.Password, err = utils.EncryptAESBase64(b.GetId(), conf.Password)
		if err != nil {
			return err
		}
		info = &api.HostLoginInfo{
			Username: conf.Username,
			Password: conf.Password,
			Ip:       conf.RemoteIP,
		}
	} else {
		info = &api.HostLoginInfo{
			Username: "None",
			Password: "None",
			Ip:       "None",
		}
	}

	data := info.JSON(info)
	_, err := modules.Hosts.SetMetadata(session, b.GetId(), data)
	return err
}

func (b *SBaremetalInstance) SyncStatusBackground() {
	go func() {
		b.AutoSyncAllStatus(context.Background())
	}()
}

func (b *SBaremetalInstance) InitializeServer(s *mcclient.ClientSession, name string) error {
	params := jsonutils.NewDict()
	params.Set("name", jsonutils.NewString(name))
	_, err := modules.Hosts.PerformAction(s, b.GetId(), "initialize", params)
	return err
}

func (b *SBaremetalInstance) ServerLoadDesc(ctx context.Context) error {
	res, err := modules.Hosts.Get(b.manager.GetClientSession(), b.GetId(), nil)
	if err != nil {
		return err
	}
	b.SaveDesc(ctx, res)
	sid, err := res.GetString("server_id")
	if err == nil {
		sDesc := jsonutils.NewDict()
		sDesc.Set("uuid", jsonutils.NewString(sid))
		b.server, err = newBaremetalServer(b, sDesc)
		return err
	} else {
		return nil
	}
}

func PowerStatusToBaremetalStatus(status types.PowerStatus) string {
	switch status {
	case types.POWER_STATUS_ON:
		return baremetalstatus.RUNNING
	case types.POWER_STATUS_OFF:
		return baremetalstatus.READY
	}
	return baremetalstatus.UNKNOWN
}

func PowerStatusToServerStatus(bm *SBaremetalInstance, status types.PowerStatus) string {
	switch status {
	case types.POWER_STATUS_ON:
		if conf, _ := bm.GetSSHConfig(); conf == nil {
			return baremetalstatus.SERVER_RUNNING
		} else {
			if !bm.HasBMC() && !bm.IsMaintenance() {
				return baremetalstatus.SERVER_READY
			} else {
				return baremetalstatus.SERVER_ADMIN
			}
		}
	case types.POWER_STATUS_OFF:
		return baremetalstatus.SERVER_READY
	}
	return baremetalstatus.UNKNOWN
}

func (b *SBaremetalInstance) AutoSyncStatus(ctx context.Context) {
	b.SyncStatus(ctx, "", "")
}

func (b *SBaremetalInstance) SyncStatus(ctx context.Context, status string, reason string) {
	if status == "" {
		powerStatus, err := b.GetPowerStatus()
		if err != nil {
			log.Errorf("Get power status error: %v", err)
		}
		status = PowerStatusToBaremetalStatus(powerStatus)
	}
	b.desc.Set("status", jsonutils.NewString(status))
	b.AutoSaveDesc(ctx)
	params := jsonutils.NewDict()
	params.Add(jsonutils.NewString(status), "status")
	if reason != "" {
		params.Add(jsonutils.NewString(reason), "reason")
	}
	_, err := modules.Hosts.PerformAction(b.GetClientSession(), b.GetId(), "status", params)
	if err != nil {
		log.Errorf("Update baremetal %s status %s error: %v", b.GetId(), status, err)
		return
	}
	log.Infof("Update baremetal %s to status %s", b.GetId(), status)
}

func (b *SBaremetalInstance) AutoSyncAllStatus(ctx context.Context) {
	b.SyncAllStatus(ctx, "")
}

func (b *SBaremetalInstance) DelayedSyncStatus(ctx context.Context, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
	b.AutoSyncAllStatus(ctx)
	return nil, nil
}

func (b *SBaremetalInstance) SyncAllStatus(ctx context.Context, status types.PowerStatus) {
	var err error
	if status == "" {
		status, err = b.GetPowerStatus()
		if err != nil {
			log.Errorf("Get power status error: %v", err)
		}
	}
	b.SyncStatus(ctx, PowerStatusToBaremetalStatus(status), "SyncAllStatus by baremetal-agent")
	b.SyncServerStatus(status)
}

func (b *SBaremetalInstance) SyncServerStatus(powerStatus types.PowerStatus) {
	if b.GetServerId() == "" {
		return
	}
	status := PowerStatusToServerStatus(b, powerStatus)
	if powerStatus == "" {
		powerStatus, err := b.GetPowerStatus()
		if err != nil {
			log.Errorf("Get power status error: %v", err)
		}
		status = PowerStatusToServerStatus(b, powerStatus)
	}
	params := &api.ServerPerformStatusInput{}
	params.Status = status
	params.PowerStates = string(powerStatus)
	_, err := modules.Servers.PerformAction(b.GetClientSession(), b.GetServerId(), "status", jsonutils.Marshal(params))
	if err != nil {
		log.Errorf("Update server %s status %s error: %v", b.GetServerName(), status, err)
		return
	}
	log.Infof("Update server %s to status %s", b.GetServerName(), status)
}

func (b *SBaremetalInstance) GetNics() []types.SNic {
	nics := []types.SNic{}
	err := b.desc.Unmarshal(&nics, "nic_info")
	if err != nil {
		log.Errorf("Unmarshal desc to get nics error: %v", err)
		return nil
	}
	return nics
}

func (b *SBaremetalInstance) getNicByType(nicType compute.TNicType) *types.SNic {
	nics := b.GetNics()
	if len(nics) == 0 {
		return nil
	}
	for i := range nics {
		if nics[i].Type == nicType {
			return &nics[i]
		}
	}
	return nil
}

func (b *SBaremetalInstance) GetNicByMac(mac net.HardwareAddr) *types.SNic {
	nics := b.GetNics()
	if len(nics) == 0 {
		return nil
	}
	for _, nic := range nics {
		tmp := nic
		if tmp.Mac == mac.String() {
			return &tmp
		}
	}
	return nil
}

func (b *SBaremetalInstance) GetAdminNic() *types.SNic {
	return b.getNicByType(api.NIC_TYPE_ADMIN)
}

func (b *SBaremetalInstance) NeedPXEBoot() bool {
	task := b.GetTask()
	taskName := "nil"
	serverId := b.GetServerId()
	if task != nil {
		taskName = task.GetName()
	}
	taskNeedPXEBoot := false
	if task != nil && task.NeedPXEBoot() {
		taskNeedPXEBoot = true
	}
	ret := false
	reason := ""
	if taskNeedPXEBoot {
		reason = fmt.Sprintf("Task %s need PXE boot", taskName)
	}
	emptyBmNoTask := false
	if task == nil && len(serverId) == 0 && b.GetHostType() == "baremetal" {
		emptyBmNoTask = true
		reason = fmt.Sprintf("Baremetal %s is empty and no task executing", b.GetName())
	}
	noTaskInMaintenance := false
	if task == nil && b.IsMaintenance() {
		noTaskInMaintenance = true
		reason = fmt.Sprintf("Baremetal %s no task executing but in maintenance", b.GetName())
	}
	if taskNeedPXEBoot || emptyBmNoTask || noTaskInMaintenance {
		ret = true
	}
	log.Infof("Check task %s, server %s NeedPXEBoot: %v(%s)", taskName, serverId, ret, reason)
	return ret
}

func (b *SBaremetalInstance) IsMaintenance() bool {
	isMt, _ := b.desc.Bool("is_maintenance")
	return isMt
}

func (b *SBaremetalInstance) GetArch() (string, error) {
	cpuArch, err := b.desc.GetString("cpu_architecture")
	if err != nil {
		return "", errors.Wrap(err, "Get cpu_architecture from desc")
	}
	if cpuArch == "" {
		return "", errors.Errorf("cpu_architecture is empty")
	}
	return cpuArch, nil
}

func (b *SBaremetalInstance) GetHostType() string {
	hostType, _ := b.desc.GetString("host_type")
	return hostType
}

func (b *SBaremetalInstance) GetIPMINic(cliMac net.HardwareAddr) *types.SNic {
	nic := b.getNicByType(api.NIC_TYPE_IPMI)
	if nic == nil {
		return nil
	}
	if nic.Mac == cliMac.String() {
		return nic
	}
	return nil
}

func (b *SBaremetalInstance) GetIPMINicIPAddr() string {
	nic := b.getNicByType(api.NIC_TYPE_IPMI)
	if nic == nil {
		return ""
	}
	return nic.IpAddr
}

func (b *SBaremetalInstance) GetDHCPConfig(cliMac net.HardwareAddr) (*dhcp.ResponseConfig, error) {
	var nic *types.SNic
	var hostname string
	var osName string
	if b.GetServer() != nil && (b.GetTask() == nil || !b.GetTask().NeedPXEBoot()) {
		nic = b.GetServer().GetNicByMac(cliMac)
		hostname = b.GetServer().GetHostName()
		osName = b.GetServer().GetOsName()
	} else {
		nic = b.GetNicByMac(cliMac)
	}
	if nic == nil {
		return nil, fmt.Errorf("GetNicDHCPConfig no nic found by mac: %s", cliMac)
	}
	return b.getDHCPConfig(nic, hostname, false, 0, osName)
}

func (b *SBaremetalInstance) GetPXEDHCPConfig(arch uint16) (*dhcp.ResponseConfig, error) {
	return b.getDHCPConfig(b.GetAdminNic(), "", true, arch, "Linux")
}

func (b *SBaremetalInstance) getDHCPConfig(
	nic *types.SNic,
	hostName string,
	isPxe bool,
	arch uint16,
	osName string,
) (*dhcp.ResponseConfig, error) {
	if hostName == "" {
		hostName = b.GetName()
	}
	if osName == "" {
		osName = "Linux"
	}
	serverIP, err := b.manager.Agent.GetDHCPServerIP()
	if err != nil {
		return nil, err
	}
	if o.Options.BootLoader == o.BOOT_LOADER_SYSLINUX && arch != dhcp.CLIENT_ARCH_EFI_ARM64 {
		if isPxe && dhcp.IsUEFIPxeArch(arch) && !b.NeedPXEBoot() {
			// TODO: use chainloader boot UEFI firmware,
			// currently not response PXE request,
			// and let BIOS detect bootable device
			b.ClearSSHConfig()
			return nil, errors.Errorf("Baremetal %s not need UEFI PXE boot", b.GetName())
		}
	}
	return GetNicDHCPConfig(nic, serverIP, b.manager.Agent.ListenInterface.HardwareAddr, hostName, isPxe, arch, osName)
}

func (b *SBaremetalInstance) GetNotifyUrl() string {
	return fmt.Sprintf("%s/baremetals/%s/notify", b.manager.Agent.GetListenUri(), b.GetId())
}

func (b *SBaremetalInstance) getHTTPEndpoint() (string, error) {
	serverIP, err := b.manager.Agent.GetDHCPServerIP()
	if err != nil {
		return "", errors.Wrap(err, "GetDHCPServerIP")
	}
	return fmt.Sprintf("%s:%d", serverIP, o.Options.Port+1000), nil
}

func (b *SBaremetalInstance) getHTTPFileUrl(filename string) string {
	endpoint, err := b.getHTTPEndpoint()
	if err != nil {
		log.Errorf("Get http file server endpoint: %v", err)
		return filename
	}
	return fmt.Sprintf("http://%s/tftp/%s", endpoint, filename)
}

func (b *SBaremetalInstance) GetImageUrl(disableImageCache bool) string {
	if disableImageCache {
		url, err := b.GetPublicClientSession().GetServiceURL(apis.SERVICE_TYPE_IMAGE, apiidenty.EndpointInterfacePublic, httputils.GET)
		if err != nil {
			log.Errorf("Get image public url: %v", err)
			return ""
		}
		return url
	}
	serverIP, err := b.manager.Agent.GetDHCPServerIP()
	if err != nil {
		log.Errorf("Get http file server: %v", err)
		return ""
	}
	// no /images/, rootcreate.sh will add this
	return fmt.Sprintf("http://%s:%d", serverIP, o.Options.Port+1000)
}

func (b *SBaremetalInstance) getBootIsoUrl() string {
	serverIP, err := b.manager.Agent.GetDHCPServerIP()
	if err != nil {
		log.Errorf("Get http file server: %v", err)
		return ""
	}
	// no /images/, rootcreate.sh will add this
	return fmt.Sprintf("http://%s:%d/bootiso/%s.iso", serverIP, o.Options.Port+1000, b.GetId())
}

func (b *SBaremetalInstance) GetTFTPResponse() string {
	arch, err := b.GetArch()
	if err != nil {
		log.Errorf("get arch error: %v", err)
	}
	if o.Options.BootLoader == o.BOOT_LOADER_SYSLINUX && arch != apis.OS_ARCH_AARCH64 {
		return b.getSyslinuxConf(true)
	}
	return b.getGrubPXEConf(true)
}

func (b *SBaremetalInstance) getIsolinuxConf() string {
	return b.getSyslinuxConf(false)
}

func (b *SBaremetalInstance) getSyslinuxPath(filename string, isTftp bool) string {
	if isTftp {
		return b.getHTTPFileUrl(filename)
	} else {
		return filename
	}
}

func (b *SBaremetalInstance) findAccessNetwork(accessIp string) (*types.SNetworkConfig, error) {
	params := jsonutils.NewDict()
	params.Add(jsonutils.NewString(accessIp), "ip")
	params.Add(jsonutils.NewString("system"), "scope")
	params.Add(jsonutils.JSONTrue, "is_classic")
	session := b.manager.GetClientSession()
	ret, err := modules.Networks.List(session, params)
	if err != nil {
		return nil, err
	}
	if len(ret.Data) == 0 {
		return nil, errors.Wrapf(httperrors.ErrNotFound, "accessIp %s", accessIp)
	}
	network := types.SNetworkConfig{}
	err = ret.Data[0].Unmarshal(&network)
	return &network, err
}

func (b *SBaremetalInstance) getKernelArgs(isTftp bool, initramfs string) string {
	args := []string{
		fmt.Sprintf("token=%s", auth.GetTokenString()),
		fmt.Sprintf("url=%s", b.GetNotifyUrl()),
	}
	bootmode := api.BOOT_MODE_PXE
	if !isTftp {
		adminNic := b.GetAdminNic()
		var mac string
		var addr string
		var mask string
		var gateway string
		if adminNic != nil {
			mac = adminNic.GetMac().String()
			addr = adminNic.IpAddr
			mask = adminNic.GetNetMask()
			gateway = adminNic.Gateway
		} else {
			accessIp := b.GetAccessIp()
			accessNet, _ := b.findAccessNetwork(accessIp)
			if accessNet != nil {
				addr = accessIp
				mask = netutils.Masklen2Mask(int8(accessNet.GuestIpMask)).String()
				gateway = accessNet.GuestGateway
			}
		}
		serverIP, _ := b.manager.Agent.GetDHCPServerIP()
		args = append(args, fmt.Sprintf("mac=%s", mac))
		args = append(args, fmt.Sprintf("dest=%s", serverIP))
		args = append(args, fmt.Sprintf("gateway=%s", gateway))
		args = append(args, fmt.Sprintf("addr=%s", addr))
		args = append(args, fmt.Sprintf("mask=%s", mask))
		bootmode = api.BOOT_MODE_ISO
	}
	args = append(args, fmt.Sprintf("bootmod=%s", bootmode))
	return strings.Join(args, " ")
}

func (b *SBaremetalInstance) getGrubPXEConf(isTftp bool) string {
	arch, err := b.GetArch()
	if err != nil {
		log.Warningf("Get baremetal %s architecture error: %v", b.GetName(), err)
		arch = apis.OS_ARCH_X86_64
	}
	kernel := "kernel"
	initrd := "initramfs"
	if arch == apis.OS_ARCH_AARCH64 {
		kernel = "kernel_aarch64"
		initrd = "initramfs_aarch64"
	}
	// TODO: support not tftp situation
	kernelArgs := b.getKernelArgs(isTftp, initrd)
	if len(o.Options.NfsBootRootfs) > 0 {
		kernelArgs = fmt.Sprintf("root=/dev/nfs nfsroot=%s rw", o.Options.NfsBootRootfs)
	}
	var resp string
	endpoint, err := b.getHTTPEndpoint()
	if err != nil {
		log.Fatalf("getHTTPEndpoint %s", err)
	}
	if b.NeedPXEBoot() {
		resp = grub.GetYunionOSConfig(3, endpoint, kernel, kernelArgs, initrd, o.Options.EnableGrubTftpDownload)
	} else {
		resp = grub.GetAutoFindConfig()
		b.ClearSSHConfig()
	}
	return resp
}

func (b *SBaremetalInstance) getSyslinuxConf(isTftp bool) string {
	resp := `DEFAULT start
serial 1 115200

LABEL start
    MENU LABEL ^Start
    MENU default
`

	if b.NeedPXEBoot() {
		kernel := "vmlinuz"
		initramfs := "initrd.img"
		if isTftp {
			kernel = b.getHTTPFileUrl("kernel")
			initramfs = b.getHTTPFileUrl("initramfs")
		}
		resp += fmt.Sprintf("    kernel %s\n", kernel)
		args := []string{
			fmt.Sprintf("initrd=%s", initramfs),
			fmt.Sprintf("token=%s", auth.GetTokenString()),
			fmt.Sprintf("url=%s", b.GetNotifyUrl()),
		}
		bootmode := api.BOOT_MODE_PXE
		if !isTftp {
			adminNic := b.GetAdminNic()
			var mac string
			var addr string
			var mask string
			var gateway string
			if adminNic != nil {
				mac = adminNic.GetMac().String()
				addr = adminNic.IpAddr
				mask = adminNic.GetNetMask()
				gateway = adminNic.Gateway
			} else {
				accessIp := b.GetAccessIp()
				accessNet, _ := b.findAccessNetwork(accessIp)
				if accessNet != nil {
					addr = accessIp
					mask = netutils.Masklen2Mask(int8(accessNet.GuestIpMask)).String()
					gateway = accessNet.GuestGateway
				}
			}
			serverIP, _ := b.manager.Agent.GetDHCPServerIP()
			args = append(args, fmt.Sprintf("mac=%s", mac))
			args = append(args, fmt.Sprintf("dest=%s", serverIP))
			args = append(args, fmt.Sprintf("gateway=%s", gateway))
			args = append(args, fmt.Sprintf("addr=%s", addr))
			args = append(args, fmt.Sprintf("mask=%s", mask))
			bootmode = api.BOOT_MODE_ISO
		}
		args = append(args, fmt.Sprintf("bootmod=%s", bootmode))
		resp += fmt.Sprintf("    append %s\n", strings.Join(args, " "))
	} else {
		resp += fmt.Sprintf("    COM32 %s\n", b.getSyslinuxPath("chain.c32", isTftp))
		resp += "    APPEND hd0 0\n"
		b.ClearSSHConfig()
	}
	// log.Debugf("[SysLinux config]: \n%s", resp)
	return resp
}

func (b *SBaremetalInstance) GetTaskQueue() *tasks.TaskQueue {
	return b.taskQueue
}

func (b *SBaremetalInstance) GetTask() tasks.ITask {
	return b.taskQueue.GetTask()
}

func (b *SBaremetalInstance) SetTask(task tasks.ITask) {
	// hack: clear exist tasks if task is server destroy task
	if task.GetName() == tasks.BAREMETAL_SERVER_DESTROY_TASK {
		log.Infof("clear tasks of baremetal %s before executing %s", b.GetName(), task.GetName())
		b.taskQueue.ClearTasks()
	}
	b.taskQueue.AppendTask(task)
	if reflect.DeepEqual(task, b.taskQueue.GetTask()) {
		log.Infof("Execute task %s of baremetal %s", task.GetName(), b.GetName())
		tasks.ExecuteTask(task, nil)
	} else {
		log.Warningf("Append task %s of baremetal %s before executing %s", task.GetName(), b.GetName(), b.taskQueue.DebugString())
	}
}

func (b *SBaremetalInstance) InitAdminNetif(
	ctx context.Context,
	cliMac net.HardwareAddr,
	wireId string,
	nicType compute.TNicType,
	netType api.TNetworkType,
	isDoImport bool,
	importIpAddr string,
) error {
	// start prepare task
	// sync status to PREPARE
	if !isDoImport && nicType == api.NIC_TYPE_ADMIN &&
		utils.IsInStringArray(b.GetStatus(),
			[]string{baremetalstatus.INIT,
				baremetalstatus.PREPARE,
				baremetalstatus.PREPARE_FAIL,
				baremetalstatus.UNKNOWN}) &&
		b.GetTask() == nil && b.GetServer() == nil {
		b.SetTask(tasks.NewBaremetalServerPrepareTask(b))
		b.SyncStatus(ctx, baremetalstatus.PREPARE, "")
	}

	nic := b.GetNicByMac(cliMac)
	if nic == nil || nic.WireId == "" {
		_, err := b.attachWire(cliMac, wireId, nicType)
		if err != nil {
			return err
		}
		return b.postAttachWire(ctx, cliMac, nicType, netType, importIpAddr)
	} else if nic.IpAddr == "" {
		return b.postAttachWire(ctx, cliMac, nicType, netType, importIpAddr)
	}
	return nil
}

func (b *SBaremetalInstance) RegisterNetif(ctx context.Context, cliMac net.HardwareAddr, wireId string) error {
	var nicType compute.TNicType
	nic := b.GetNicByMac(cliMac)
	if nic != nil {
		nicType = nic.Type
	}
	if nic == nil || nic.WireId == "" || nic.WireId != wireId {
		desc, err := b.attachWire(cliMac, wireId, nicType)
		if err != nil {
			return err
		}
		return b.SaveDesc(ctx, desc)
	}
	return nil
}

func (b *SBaremetalInstance) attachWire(mac net.HardwareAddr, wireId string, nicType compute.TNicType) (jsonutils.JSONObject, error) {
	session := b.manager.GetClientSession()
	params := jsonutils.NewDict()
	params.Add(jsonutils.NewString(mac.String()), "mac")
	if nicType != "" {
		params.Add(jsonutils.NewString(string(nicType)), "nic_type")
	}
	params.Add(jsonutils.NewString(wireId), "wire")
	params.Add(jsonutils.NewInt(-1), "index")
	params.Add(jsonutils.JSONTrue, "link_up")
	return modules.Hosts.PerformAction(session, b.GetId(), "add-netif", params)
}

func (b *SBaremetalInstance) postAttachWire(ctx context.Context, mac net.HardwareAddr, nicType compute.TNicType, netType api.TNetworkType, ipAddr string) error {
	if ipAddr == "" {
		switch nicType {
		case api.NIC_TYPE_IPMI:
			oldIPMIConf := b.GetRawIPMIConfig()
			if oldIPMIConf != nil && oldIPMIConf.IpAddr != "" {
				ipAddr = oldIPMIConf.IpAddr
			}
		case api.NIC_TYPE_ADMIN:
			accessIp := b.GetAccessIp()
			if accessIp != "" {
				ipAddr = accessIp
			}
		}
	}
	desc, err := b.enableWire(mac, ipAddr, nicType, netType)
	if err != nil {
		return err
	}
	return b.SaveDesc(ctx, desc)
}

func (b *SBaremetalInstance) enableWire(mac net.HardwareAddr, ipAddr string, nicType compute.TNicType, netType api.TNetworkType) (jsonutils.JSONObject, error) {
	session := b.manager.GetClientSession()
	params := jsonutils.NewDict()
	params.Add(jsonutils.NewString(mac.String()), "mac")
	if nicType != "" {
		params.Add(jsonutils.NewString(string(nicType)), "nic_type")
	}
	if ipAddr != "" {
		params.Add(jsonutils.NewString(ipAddr), "ip_addr")
		params.Add(jsonutils.JSONTrue, "reserve")
	}
	if nicType == api.NIC_TYPE_IPMI {
		params.Add(jsonutils.NewString("stepup"), "alloc_dir") // alloc bottom up
	}
	if len(netType) > 0 {
		params.Add(jsonutils.NewString(string(netType)), "net_type")
	}
	log.Infof("enable net if params: %s", params.String())
	return modules.Hosts.PerformAction(session, b.GetId(), "enable-netif", params)
}

func (b *SBaremetalInstance) GetIPMIConfig() *types.SIPMIInfo {
	conf := b.GetRawIPMIConfig()
	if conf == nil {
		log.Debugf("GetIPMIConfig conf is nil")
		return nil
	}
	if conf.Password == "" {
		log.Debugf("GetIPMIConfig password is nil")
		return nil
	}
	if conf.Username == "" && b.profile != nil {
		conf.Username = b.profile.RootName
	}
	if conf.IpAddr == "" {
		nicIPAddr := b.GetIPMINicIPAddr()
		if nicIPAddr != "" {
			conf.IpAddr = nicIPAddr
		}
	}
	conf.Password = utils.Unquote(conf.Password) // XXX: remove quotes!!!
	if conf.IpAddr == "" {
		log.Debugf("GetIPMIConfig ipaddr s nil")
		return nil
	}
	return conf
}

func (b *SBaremetalInstance) GetRawIPMIConfig() *types.SIPMIInfo {
	ipmiInfo := types.SIPMIInfo{}
	err := b.desc.Unmarshal(&ipmiInfo, "ipmi_info")
	if err != nil {
		log.Errorf("Unmarshal IPMIInfo error: %v", err)
		return nil
	}
	if ipmiInfo.Password != "" {
		ipmiInfo.Password, err = utils.DescryptAESBase64(b.GetId(), ipmiInfo.Password)
		if err != nil {
			log.Errorf("DescryptAESBase64 IPMI password error: %v", err)
			return nil
		}
	}
	return &ipmiInfo
}

func (b *SBaremetalInstance) GetUEFIInfo() (*types.EFIBootMgrInfo, error) {
	key := "uefi_info"
	if !b.desc.Contains(key) {
		return nil, nil
	}
	info := new(types.EFIBootMgrInfo)
	if err := b.desc.Unmarshal(info, key); err != nil {

		return nil, errors.Wrap(err, "Unmarshal json")
	}
	return info, nil
}

func (b *SBaremetalInstance) GetAccessIp() string {
	accessIp, _ := b.desc.GetString("access_ip")
	return accessIp
}

func (b *SBaremetalInstance) GetAccessMac() string {
	accessMac, _ := b.desc.GetString("access_mac")
	return accessMac
}

func (b *SBaremetalInstance) GetServer() baremetaltypes.IBaremetalServer {
	b.serverLock.Lock()
	defer b.serverLock.Unlock()
	if !b.desc.Contains("server_id") && b.server != nil {
		log.Warningf("baremetal %s server_id not present, remove server %q", b.GetName(), b.server.GetName())
		b.removeServer()
		return nil
	}
	return b.server
}

func (b *SBaremetalInstance) GetServerId() string {
	srv := b.GetServer()
	if srv == nil {
		return ""
	}
	return srv.GetId()
}

func (b *SBaremetalInstance) GetServerName() string {
	srv := b.GetServer()
	if srv == nil {
		return ""
	}
	return srv.GetName()
}

func (b *SBaremetalInstance) RemoveServer() {
	b.serverLock.Lock()
	defer b.serverLock.Unlock()
	b.removeServer()
}

func (b *SBaremetalInstance) removeServer() {
	if b.server != nil {
		b.server.RemoveDesc()
		b.server = nil
	}
}

func (b *SBaremetalInstance) SetExistingIPMIIPAddr(ipAddr string) {
	info, _ := b.desc.Get("ipmi_info")
	if info == nil {
		info = jsonutils.NewDict()
	}
	oIPAddr, _ := info.GetString("ip_addr")
	if oIPAddr == "" {
		info.(*jsonutils.JSONDict).Add(jsonutils.NewString(ipAddr), "ip_addr")
	}
	b.desc.Set("ipmi_info", info)
}

func (b *SBaremetalInstance) HasBMC() bool {
	conf := b.GetIPMIConfig()
	if conf == nil {
		return false
	}
	return true
}

func (b *SBaremetalInstance) GetHostSSHClient() (*ssh.Client, error) {
	conf, err := b.GetSSHConfig()
	if err != nil {
		return nil, errors.Wrap(err, "Get host ssh config")
	}
	if conf == nil {
		return nil, errors.Errorf("Host ssh config is empty")
	}
	sshCli, err := ssh.NewClient(conf.RemoteIP, 22, "root", conf.Password, "")
	if err != nil {
		return nil, errors.Wrap(err, "New ssh client")
	}
	return sshCli, nil
}

func (b *SBaremetalInstance) GetServerSSHClient() (*ssh.Client, error) {
	s := b.GetServer()
	if s == nil {
		return nil, errors.Error("No server")
	}

	privateKey, err := modules.Sshkeypairs.FetchPrivateKey(context.TODO(), auth.AdminCredential())
	if err != nil {
		return nil, errors.Wrapf(err, "Get server %s login info", s.GetId())
	}
	nics := s.GetNics()
	var errs []error
	for idx, nic := range nics {
		if nic.Ip != "" {
			for _, user := range []string{"cloudroot", "root"} {
				sshCli, err := ssh.NewClient(nic.Ip, 22, user, "", privateKey)
				if err != nil {
					err = errors.Wrapf(err, "New server %s ssh client %s@%s", s.GetName(), user, nic.Ip)
					errs = append(errs, err)
				} else {
					return sshCli, nil
				}
			}
		} else {
			errs = append(errs, errors.Errorf("nic %d link_up: %v, ip: %q", idx, nic.LinkUp, nic.Ip))
		}
	}

	return nil, errors.NewAggregate(errs)
}

func (b *SBaremetalInstance) SSHReachable() (bool, error) {
	var errs []error
	if cli, err := b.GetHostSSHClient(); err != nil {
		errs = append(errs, err)
	} else {
		// host ssh reachable
		cli.Close()
		return true, nil
	}
	if cli, err := b.GetServerSSHClient(); err != nil {
		errs = append(errs, err)
	} else {
		// server ssh reachable
		cli.Close()
		return true, nil
	}
	return false, errors.NewAggregate(errs)
}

func (b *SBaremetalInstance) sshRunWrapper(
	hostRun func(hostCli *ssh.Client) ([]string, error),
	serverRun func(serverCli *ssh.Client) ([]string, error),
) ([]string, error) {
	hostCli, err := b.GetHostSSHClient()
	if err != nil {
		log.Warningf("Get host ssh client error: %v", err)
	} else {
		if hostCli != nil {
			return hostRun(hostCli)
		}
	}

	serverCli, err := b.GetServerSSHClient()
	if err != nil {
		return nil, errors.Wrapf(err, "Get baremetal %s server ssh client", b.GetName())
	}
	return serverRun(serverCli)
}

func (b *SBaremetalInstance) sshRun(hostCmd string, serverCmd string) ([]string, error) {
	return b.sshRunWrapper(
		func(hostCli *ssh.Client) ([]string, error) {
			return hostCli.RawRun(hostCmd)
		},
		func(serverCli *ssh.Client) ([]string, error) {
			return serverCli.RunWithTTY(serverCmd)
		},
	)
}

func (b *SBaremetalInstance) adjustUEFIWrapper(ctx context.Context, cli *ssh.Client, f func() error) error {
	isUEFI, err := uefi.RemoteIsUEFIBoot(cli)
	if err != nil {
		return errors.Wrap(err, "Check is uefi boot")
	}
	if !isUEFI {
		return b.CleanUEFIInfo(ctx)
	}
	return f()
}

func (b *SBaremetalInstance) AdjustUEFICurrentBootOrder(ctx context.Context, hostCli *ssh.Client) error {
	return b.adjustUEFIWrapper(ctx, hostCli, func() error {
		mgr, err := uefi.NewEFIBootMgrFromRemote(hostCli, false)
		if err != nil {
			return errors.Wrap(err, "NewEFIBootMgrFromRemote")
		}
		if err := uefi.RemoteSetCurrentBootAtFirst(hostCli, mgr); err != nil {
			return errors.Wrap(err, "Set current boot order at first")
		}
		return b.SendUEFIInfo(ctx, mgr)
	})
}

func (b *SBaremetalInstance) updateUEFIInfo(ctx context.Context, uefiData jsonutils.JSONObject) error {
	desc := b.desc
	desc.Add(uefiData, "uefi_info")
	if err := b.SaveDesc(ctx, desc); err != nil {
		return errors.Wrap(err, "Save uefi_info")
	}
	updateData := jsonutils.NewDict()
	updateData.Add(uefiData, "uefi_info")
	if _, err := modules.Hosts.Update(b.GetClientSession(), b.GetId(), updateData); err != nil {
		return errors.Wrap(err, "Update cloud uefi info")
	}
	return nil
}

func (b *SBaremetalInstance) CleanUEFIInfo(ctx context.Context) error {
	if err := b.updateUEFIInfo(ctx, jsonutils.NewDict()); err != nil {
		return errors.Wrap(err, "CleanUEFIInfo")
	}
	return nil
}

func (b *SBaremetalInstance) SendUEFIInfo(ctx context.Context, mgr *uefi.BootMgr) error {
	info, err := mgr.ToEFIBootMgrInfo()
	if err != nil {
		return err
	}
	if err := b.updateUEFIInfo(ctx, jsonutils.Marshal(info)); err != nil {
		return errors.Wrap(err, "SendUEFIInfo")
	}
	return nil
}

func (b *SBaremetalInstance) adjustServerUEFIBootOrder(ctx context.Context, srvCli *ssh.Client) error {
	return b.adjustUEFIWrapper(ctx, srvCli, func() error {
		info, err := b.GetUEFIInfo()
		if err != nil {
			return errors.Wrap(err, "GetUEFIInfo from local desc")
		}
		if info == nil {
			return uefi.RemoteTryToSetPXEBoot(srvCli)
		}
		if err := uefi.RemoteTryToSetPXEBoot(srvCli); err != nil {
			log.Warningf("RemoteTryToSetPXEBoot error: %v", err)
		}
		_, err = uefi.RemoteSetBootOrderByInfo(srvCli, info.GetPXEEntry())
		return err
	})
}

func (b *SBaremetalInstance) AdjustUEFIBootOrder(ctx context.Context) error {
	_, err := b.sshRunWrapper(
		func(hostCli *ssh.Client) ([]string, error) {
			return nil, b.AdjustUEFICurrentBootOrder(ctx, hostCli)
		},
		func(srvCli *ssh.Client) ([]string, error) {
			return nil, b.adjustServerUEFIBootOrder(ctx, srvCli)
		},
	)
	return err
}

func (b *SBaremetalInstance) SSHReboot(ctx context.Context) error {
	if !b.HasBMC() {
		// try adjust uefi boot order before reboot
		if err := b.AdjustUEFIBootOrder(ctx); err != nil {
			return errors.Wrap(err, "Adjust uefi boot order")
		}
	}
	if _, err := b.sshRun("/sbin/reboot", "sudo shutdown -r now && exit"); err != nil {
		if !ssh.IsExitMissingError(err) {
			return errors.Wrap(err, "Try reboot")
		}
	}
	b.ClearSSHConfig()
	return nil
}

func (b *SBaremetalInstance) SSHShutdown() error {
	if _, err := b.sshRun("/sbin/poweroff", "sudo shutdown -h now && exit"); err != nil {
		if ssh.IsExitMissingError(err) {
			return nil
		}
		return errors.Wrap(err, "Try poweroff")
	}
	return nil
}

func (b *SBaremetalInstance) GetIPMITool() *ipmitool.LanPlusIPMI {
	conf := b.GetIPMIConfig()
	if conf == nil {
		log.Debugf("GetIPMIConfig is nil")
		return nil
	}
	return ipmitool.NewLanPlusIPMI(conf.IpAddr, conf.Username, conf.Password)
}

func (b *SBaremetalInstance) isRedfishCapable() bool {
	conf := b.GetIPMIConfig()
	if conf == nil {
		return false
	}
	if !conf.Verified {
		return false
	}
	if !conf.RedfishApi {
		return false
	}
	return true
}

func (b *SBaremetalInstance) GetRedfishCli(ctx context.Context) redfish.IRedfishDriver {
	if !b.isRedfishCapable() {
		return nil
	}
	conf := b.GetIPMIConfig()
	var endpoint = "https://" + conf.IpAddr
	if strings.Contains(conf.IpAddr, ":") {
		endpoint = fmt.Sprintf("https://[%s]", conf.IpAddr)
	}
	return redfish.NewRedfishDriver(ctx, endpoint, conf.Username, conf.Password, false)
}

func (b *SBaremetalInstance) GetIPMILanChannel() uint8 {
	conf := b.GetIPMIConfig()
	if conf == nil {
		return 0
	}
	return conf.LanChannel
}

func (b *SBaremetalInstance) DoPXEBoot() error {
	log.Infof("Do PXE Boot ........., wait")
	b.ClearSSHConfig()
	ipmiCli := b.GetIPMITool()
	if ipmiCli != nil {
		return ipmitool.DoRebootToPXE(ipmiCli)
	}
	return fmt.Errorf("Baremetal %s ipmitool is nil", b.GetId())
}

func (b *SBaremetalInstance) doRedfishPowerOn() error {
	log.Infof("Do Redfish PowerOn ........., wait")
	ctx := context.Background()
	b.ClearSSHConfig()
	redfishApi := b.GetRedfishCli(ctx)
	if redfishApi != nil {
		err := redfishApi.Reset(ctx, "On")
		if err != nil {
			if httputils.ErrorCode(err) == 409 {
				return errors.Wrap(err, "redfishApi.Reset On fail, code 409")
			} else {
				return errors.Wrap(err, "redfishApi.Reset On")
			}
		}
		return nil
	}
	return fmt.Errorf("Baremetal %s redfishApi is nil", b.GetId())
}

func (b *SBaremetalInstance) DoRedfishPowerOn() error {
	err := b.doRedfishPowerOn()
	if err == nil {
		return nil
	}
	var errs = []error{err}
	log.Warningf("Do redfish power on error: %v, try use fallback IPMI way", err)
	ipmiCli := b.GetIPMITool()
	if err := ipmitool.DoReboot(ipmiCli); err != nil {
		errs = append(errs, errors.Wrap(err, "Fallback to use ipmitool reboot"))
	} else {
		return nil
	}
	return errors.NewAggregate(errs)
}

/*
func (b *SBaremetalInstance) DoDiskBoot() error {
	log.Infof("Do DISK Boot ........., wait")
	b.ClearSSHConfig()
	ipmiCli := b.GetIPMITool()
	if ipmiCli != nil {
		return ipmitool.DoRebootToDisk(ipmiCli)
	}
	return fmt.Errorf("Baremetal %s ipmitool is nil", b.GetId())
}
*/

func (b *SBaremetalInstance) GetPowerStatus() (types.PowerStatus, error) {
	status, err := b.getPowerStatus()
	if err != nil {
		if errors.Cause(err) != types.ErrIPMIToolNull {
			return "", errors.Wrap(err, "GetPowerStatus")
		} else if b.HasBMC() {
			return "", errors.Wrap(err, "GetPowerStatus from ipmi")
		}
	}
	return status, nil
}

func (b *SBaremetalInstance) getPowerStatus() (types.PowerStatus, error) {
	ipmiCli := b.GetIPMITool()
	if ipmiCli == nil {
		if cli, err := b.GetHostSSHClient(); err == nil {
			cli.Close()
			return types.POWER_STATUS_ON, nil
		} else {
			log.Warningf("Use host %s ssh client get powerstatus: %v", b.GetName(), err)
		}
		if cli, err := b.GetServerSSHClient(); err == nil {
			cli.Close()
			b.ClearSSHConfig()
			return types.POWER_STATUS_ON, nil
		} else {
			log.Warningf("Use server %s ssh client get powerstatus: %v", b.GetServerName(), err)
		}
		return "", errors.Wrapf(types.ErrIPMIToolNull, "Baremetal %s", b.GetId())
	}
	cps, err := ipmitool.GetChassisPowerStatus(ipmiCli)
	if err != nil {
		return "", errors.Wrap(err, "ipmitool.GetChassisPowerStatus")
	}
	return types.PowerStatus(cps), nil
}

func (b *SBaremetalInstance) DoPowerShutdown(soft bool) error {
	b.ClearSSHConfig()
	ipmiCli := b.GetIPMITool()
	if ipmiCli != nil {
		if soft {
			return ipmitool.DoSoftShutdown(ipmiCli)
		}
		return ipmitool.DoHardShutdown(ipmiCli)
	}
	return fmt.Errorf("Baremetal %s ipmitool is nil", b.GetId())
}

func (b *SBaremetalInstance) GetStorageDriver() string {
	driver, _ := b.desc.GetString("storage_driver")
	return driver
}

func (b *SBaremetalInstance) GetZoneId() string {
	return b.manager.GetZoneId()
}

func (b *SBaremetalInstance) GetZoneName() string {
	return b.manager.GetZoneName()
}

func (b *SBaremetalInstance) GetStorageCacheId() string {
	return b.manager.Agent.CacheManager.GetId()
}

func (b *SBaremetalInstance) GetBootMode() string {
	bootMode, _ := b.desc.GetString("boot_mode")
	if len(bootMode) == 0 {
		bootMode = api.BOOT_MODE_PXE
	}
	return bootMode
}

func (b *SBaremetalInstance) GetRegion() string {
	r, _ := b.desc.GetString("region")
	return r
}

func (b *SBaremetalInstance) GetRegionId() string {
	r, _ := b.desc.GetString("region_id")
	return r
}

func (b *SBaremetalInstance) GetSerialNumber() string {
	r, _ := b.desc.GetString("sn")
	return r
}

func (b *SBaremetalInstance) GetManufacture() string {
	r, _ := b.desc.GetString("sys_info", "manufacture")
	return r
}

func (b *SBaremetalInstance) GetModel() string {
	r, _ := b.desc.GetString("sys_info", "model")
	return r
}

func (b *SBaremetalInstance) GetNodeCount() string {
	r, _ := b.desc.GetString("node_count")
	return r
}

func (b *SBaremetalInstance) GetMemGb() string {
	memMb, _ := b.desc.Int("mem_size")
	return strconv.FormatInt(memMb/1024, 10)
}

func (b *SBaremetalInstance) DelayedRemove(_ context.Context, _ jsonutils.JSONObject) (jsonutils.JSONObject, error) {
	b.remove()
	return nil, nil
}

func (b *SBaremetalInstance) remove() {
	b.manager.CleanBaremetal(b.GetId())
	b.manager = nil
	b.desc = nil
}

func (b *SBaremetalInstance) StartNewTask(factory tasks.TaskFactory, userCred mcclient.TokenCredential, taskId string, data jsonutils.JSONObject) {
	go func() {
		task := factory(userCred, b, taskId, data)
		b.SetTask(task)
	}()
}

func (b *SBaremetalInstance) StartBaremetalMaintenanceTask(userCred mcclient.TokenCredential, taskId string, data jsonutils.JSONObject) {
	if jsonutils.QueryBoolean(data, "force_reboot", false) {
		b.ClearSSHConfig()
	}
	if jsonutils.QueryBoolean(data, "guest_running", false) {
		data.(*jsonutils.JSONDict).Set("soft_reboot", jsonutils.JSONTrue)
	}
	b.StartNewTask(tasks.NewBaremetalMaintenanceTask, userCred, taskId, data)
}

func (b *SBaremetalInstance) StartBaremetalUnmaintenanceTask(userCred mcclient.TokenCredential, taskId string, data jsonutils.JSONObject) {
	b.StartNewTask(tasks.NewBaremetalUnmaintenanceTask, userCred, taskId, data)
}

func (b *SBaremetalInstance) StartBaremetalReprepareTask(userCred mcclient.TokenCredential, taskId string, data jsonutils.JSONObject) {
	b.StartNewTask(tasks.NewBaremetalReprepareTask, userCred, taskId, data)
}

func (b *SBaremetalInstance) StartBaremetalResetBMCTask(userCred mcclient.TokenCredential, taskId string, data jsonutils.JSONObject) error {
	b.StartNewTask(tasks.NewBaremetalResetBMCTask, userCred, taskId, data)
	return nil
}

func (b *SBaremetalInstance) StartBaremetalIpmiProbeTask(ctx context.Context, userCred mcclient.TokenCredential, taskId string, data jsonutils.JSONObject) error {
	session := b.manager.GetClientSession()
	data, _ = b.manager.fetchBaremetal(session, b.GetId())
	if err := b.SaveDesc(ctx, data); err != nil {
		return err
	}
	b.StartNewTask(tasks.NewBaremetalIpmiProbeTask, userCred, taskId, data)
	return nil
}

func (b *SBaremetalInstance) StartBaremetalCdromTask(userCred mcclient.TokenCredential, taskId string, data jsonutils.JSONObject) error {
	b.StartNewTask(tasks.NewBaremetalCdromTask, userCred, taskId, data)
	return nil
}

func (b *SBaremetalInstance) DelayedServerReset(ctx context.Context, _ jsonutils.JSONObject) (jsonutils.JSONObject, error) {
	err := b.DoPXEBoot()
	return nil, err
}

func (b *SBaremetalInstance) StartServerCreateTask(ctx context.Context, userCred mcclient.TokenCredential, taskId string, data jsonutils.JSONObject) error {
	b.serverLock.Lock()
	defer b.serverLock.Unlock()
	if b.server != nil {
		return fmt.Errorf("Baremetal %s already have server %s", b.GetName(), b.server.GetName())
	}
	descData, err := data.Get("desc")
	if err != nil {
		return fmt.Errorf("Create data not found server desc: %v", err)
	}
	server, err := newBaremetalServer(b, descData.(*jsonutils.JSONDict))
	if err != nil {
		return fmt.Errorf("New server error: %v", err)
	}
	b.server = server
	b.desc.Set("server_id", jsonutils.NewString(b.server.GetId()))
	if err := b.AutoSaveDesc(ctx); err != nil {
		return err
	}
	b.StartNewTask(tasks.NewBaremetalServerCreateTask, userCred, taskId, data)
	return nil
}

func (b *SBaremetalInstance) StartServerDeployTask(userCred mcclient.TokenCredential, taskId string, data jsonutils.JSONObject) error {
	desc, err := data.Get("desc")
	if err != nil {
		return fmt.Errorf("Not found desc in data")
	}
	if err := b.GetServer().SaveDesc(desc); err != nil {
		return fmt.Errorf("Save server desc: %v", err)
	}
	b.StartNewTask(tasks.NewBaremetalServerDeployTask, userCred, taskId, data)
	return nil
}

func (b *SBaremetalInstance) StartServerRebuildTask(userCred mcclient.TokenCredential, taskId string, data jsonutils.JSONObject) error {
	desc, err := data.Get("desc")
	if err != nil {
		return fmt.Errorf("Not found desc in data")
	}
	if err := b.GetServer().SaveDesc(desc); err != nil {
		return fmt.Errorf("Save server desc: %v", err)
	}
	b.StartNewTask(tasks.NewBaremetalServerRebuildTask, userCred, taskId, data)
	return nil
}

func (b *SBaremetalInstance) StartServerStartTask(userCred mcclient.TokenCredential, taskId string, data jsonutils.JSONObject) error {
	b.StartNewTask(tasks.NewBaremetalServerStartTask, userCred, taskId, data)
	return nil
}

func (b *SBaremetalInstance) StartServerStopTask(userCred mcclient.TokenCredential, taskId string, data jsonutils.JSONObject) error {
	b.StartNewTask(tasks.NewBaremetalServerStopTask, userCred, taskId, data)
	return nil
}

func (b *SBaremetalInstance) StartServerDestroyTask(userCred mcclient.TokenCredential, taskId string, data jsonutils.JSONObject) {
	b.StartNewTask(tasks.NewBaremetalServerDestroyTask, userCred, taskId, data)
}

func (b *SBaremetalInstance) DelayedSyncIPMIInfo(ctx context.Context, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
	ipmiCli := b.GetIPMITool()
	lanChannel := b.GetIPMILanChannel()
	sysInfo, err := ipmitool.GetSysInfo(ipmiCli)
	if err != nil {
		return nil, errors.Wrap(err, "GetSysInfo")
	}
	profile, err := profiles.GetProfile(ctx, sysInfo)
	if err != nil {
		return nil, errors.Wrap(err, "GetProfile")
	}
	if lanChannel <= 0 {
		if len(profile.LanChannels) == 0 {
			return nil, errors.Wrap(errors.ErrInvalidStatus, "baremetal profile not valid lan channel?")
		}
		lanChannel = profile.LanChannels[0]
	}
	retObj := make(map[string]string)
	if ipAddr, _ := data.GetString("ip_addr"); ipAddr != "" {
		err = ipmitool.SetLanStaticIP(ipmiCli, lanChannel, ipAddr)
		if err != nil {
			return nil, err
		}
		// TODO: netutils.wait_ip_alive(ipAddr, 120)
		retObj["ipmi_ip_addr"] = ipAddr
	}
	if passwd, _ := data.GetString("password"); passwd != "" {
		err = ipmitool.SetLanPasswd(ipmiCli, profile.RootId, passwd)
		if err != nil {
			return nil, err
		}
		retObj["ipmi_password"] = passwd
	}
	return jsonutils.Marshal(retObj), nil
}

func (b *SBaremetalInstance) DelayedSyncDesc(ctx context.Context, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
	if data == nil {
		session := b.manager.GetClientSession()
		data, _ = b.manager.fetchBaremetal(session, b.GetId())
	}
	err := b.SaveDesc(ctx, data)
	return nil, err
}

func (b *SBaremetalInstance) DelayedServerStatus(ctx context.Context, data jsonutils.JSONObject) (jsonutils.JSONObject, error) {
	ps, err := b.GetPowerStatus()
	if err != nil {
		return nil, err
	}
	status := PowerStatusToServerStatus(b, ps)
	resp := jsonutils.NewDict()
	resp.Add(jsonutils.NewString(status), "status")
	return resp, err
}

func (b *SBaremetalInstance) SendNicInfo(ctx context.Context, nic *types.SNicDevInfo, idx int, nicType compute.TNicType, reset bool, ipAddr string, reserve bool) error {
	params := jsonutils.NewDict()
	params.Add(jsonutils.NewString(nic.Mac.String()), "mac")
	params.Add(jsonutils.NewInt(int64(nic.Speed)), "rate")
	if idx >= 0 {
		params.Add(jsonutils.NewInt(int64(idx)), "index")
	}
	if nicType != "" {
		params.Add(jsonutils.NewString(string(nicType)), "nic_type")
	}
	params.Add(jsonutils.NewInt(int64(nic.Mtu)), "mtu")
	if nic.Up != nil {
		params.Add(jsonutils.NewBool(*nic.Up), "link_up")
	}
	if reset {
		params.Add(jsonutils.JSONTrue, "reset")
	}
	if ipAddr != "" {
		params.Add(jsonutils.NewString(ipAddr), "ip_addr")
		params.Add(jsonutils.JSONTrue, "require_designated_ip")
		if reserve {
			params.Add(jsonutils.JSONTrue, "reserve")
		}
	}
	localNic := b.GetNicByMac(nic.Mac)
	if localNic != nil {
		params.Add(jsonutils.NewString(localNic.WireId), "wire_id")
	}
	resp, err := modules.Hosts.PerformAction(
		b.GetClientSession(),
		b.GetId(),
		"add-netif",
		params,
	)
	if err != nil {
		return err
	}
	return b.SaveDesc(ctx, resp)
}

func bindMount(src, dst string) error {
	err := procutils.NewCommand("touch", dst).Run()
	if err != nil {
		return errors.Wrapf(err, "touch %s", dst)
	}
	err = procutils.NewCommand("mount", "-o", "ro,bind", src, dst).Run()
	if err != nil {
		return errors.Wrapf(err, "mount %s %s", src, dst)
	}
	return nil
}

func unbindMount(dst string) error {
	err := procutils.NewCommand("umount", dst).Run()
	if err != nil {
		return errors.Wrapf(err, "umount %s", dst)
	}
	return nil
}

func (b *SBaremetalInstance) EnablePxeBoot() bool {
	return jsonutils.QueryBoolean(b.desc, "enable_pxe_boot", true)
}

func (b *SBaremetalInstance) GenerateBootISO() error {
	// precheck
	conf := b.GetRawIPMIConfig()
	if !conf.Verified {
		return errors.Error("GenerateBootISO: IPMI not supported")
	}
	if !conf.CdromBoot {
		return errors.Error("GenerateBootISO: cdrom boot not supported")
	}
	accessIp := b.GetAccessIp()
	if accessIp == "" {
		return errors.Error("GenerateBootISO: empty accessIp")
	}
	adminNic := b.GetAdminNic()
	if adminNic == nil {
		accessNet, _ := b.findAccessNetwork(accessIp)
		if accessNet == nil {
			return errors.Error("GenerateBootISO: Nil access Network")
		}
	}
	ctx := context.Background()
	redfishApi := b.GetRedfishCli(ctx)
	if redfishApi == nil {
		return errors.Wrap(httperrors.ErrNotSupported, "no valid redfishApi")
	}
	// generate ISO
	isoDir, err := ioutil.TempDir("", "bmiso")
	if err != nil {
		return errors.Wrap(err, "ioutil.TempDir")
	}
	defer os.RemoveAll(isoDir)
	isoLinDir := filepath.Join(isoDir, "isolinux")
	err = os.Mkdir(isoLinDir, os.FileMode(0766))
	if err != nil {
		return errors.Wrapf(err, "Mkdir %s", isoLinDir)
	}
	for _, f := range []string{
		"chain.c32", "ldlinux.c32", "libutil.c32", "libcom32.c32",
	} {
		err = bindMount(filepath.Join(o.Options.TftpRoot, f), filepath.Join(isoLinDir, f))
		if err != nil {
			return errors.Wrapf(err, "Link %s", f)
		}
		defer unbindMount(filepath.Join(isoLinDir, f))
	}
	for src, dst := range map[string]string{
		"kernel":    "vmlinuz",
		"initramfs": "initrd.img",
	} {
		err = bindMount(filepath.Join(o.Options.TftpRoot, src), filepath.Join(isoLinDir, dst))
		if err != nil {
			return errors.Wrapf(err, "Link %s %s", src, dst)
		}
		defer unbindMount(filepath.Join(isoLinDir, dst))
	}
	for _, f := range []string{
		"isolinux.bin",
	} {
		err = procutils.NewCommand("cp", filepath.Join(o.Options.TftpRoot, f), filepath.Join(isoLinDir, f)).Run()
		if err != nil {
			return errors.Wrapf(err, "cp %s", f)
		}
	}
	cfgCont := b.getIsolinuxConf()
	err = fileutils2.FilePutContents(filepath.Join(isoLinDir, "isolinux.cfg"), cfgCont, false)
	if err != nil {
		return errors.Wrap(err, "fileutils.FilePutContent")
	}
	args := []string{
		"-quiet",
		"-J", "-R",
		"-input-charset", "utf-8",
		"-b", "isolinux/isolinux.bin",
		"-c", "isolinux/boot.cat",
		"-no-emul-boot",
		"-boot-load-size", "4",
		"-boot-info-table",
		"-o", b.getBootIsoImagePath(),
		isoDir,
	}
	err = procutils.NewCommand("mkisofs", args...).Run()
	if err != nil {
		return errors.Wrap(err, "procutils.NewCommand mkisofs")
	}
	// mount the virtual media
	err = redfish.MountVirtualCdrom(ctx, redfishApi, b.getBootIsoUrl(), true)
	if err != nil {
		return errors.Wrap(err, "redfish.MountVirtualCdrom")
	}
	return nil
}

func (b *SBaremetalInstance) clearBootIso() error {
	ctx := context.Background()
	redfishApi := b.GetRedfishCli(ctx)
	if redfishApi == nil {
		return errors.Wrap(httperrors.ErrNotSupported, "no valid redfishApi")
	}
	err := redfish.UmountVirtualCdrom(ctx, redfishApi)
	if err != nil {
		return errors.Wrap(err, "redfish.UmountVirtualCdrom")
	}
	return b.clearBootIsoImage()
}

func (b *SBaremetalInstance) clearBootIsoImage() error {
	path := b.getBootIsoImagePath()
	if fileutils2.Exists(path) {
		return os.Remove(path)
	}
	return nil
}

func (b *SBaremetalInstance) getBootIsoImagePath() string {
	return filepath.Join(o.Options.BootIsoPath, b.GetId()+".iso")
}

func (b *SBaremetalInstance) DoNTPConfig() error {
	var urls []string
	for _, ep := range []string{"internal", "public"} {
		urls, _ = auth.GetServiceURLs("ntp", o.Options.Region, "", ep, httputils.POST)
		if len(urls) > 0 {
			break
		}
	}
	if len(urls) == 0 {
		log.Warningf("NO ntp server specified, skip DoNTPConfig")
		return nil
	}
	for i := range urls {
		if strings.HasPrefix(urls[i], "ntp://") {
			urls[i] = urls[i][6:]
		}
	}
	log.Infof("Set NTP %s", urls)
	ntpConf := redfish.SNTPConf{}
	ntpConf.ProtocolEnabled = true
	ntpConf.TimeZone = o.Options.TimeZone
	ntpConf.NTPServers = urls

	ctx := context.Background()
	redfishApi := b.GetRedfishCli(ctx)
	if redfishApi == nil {
		return errors.Wrap(httperrors.ErrNotSupported, "no valid redfishApi")
	}
	err := redfishApi.SetNTPConf(ctx, ntpConf)
	if err != nil {
		return errors.Wrap(err, "redfishApi.SetNTPConf")
	}
	return nil
}

func (b *SBaremetalInstance) fetchLogs(ctx context.Context, logType string, since time.Time) ([]redfish.SEvent, error) {
	redfishApi := b.GetRedfishCli(ctx)
	if redfishApi == nil {
		// errors.Wrap(httperrors.ErrNotSupported, "no valid redfish api")
		return nil, nil
	}
	switch logType {
	case redfish.EVENT_TYPE_SYSTEM:
		return redfishApi.ReadSystemLogs(ctx, since)
	case redfish.EVENT_TYPE_MANAGER:
		return redfishApi.ReadManagerLogs(ctx, since)
	}
	return nil, errors.Wrap(httperrors.ErrNotSupported, logType)
}

func (b *SBaremetalInstance) clearLogs(ctx context.Context, logType string) error {
	redfishApi := b.GetRedfishCli(ctx)
	if redfishApi == nil {
		return errors.Wrap(httperrors.ErrNotSupported, "no valid redfish api")
	}
	switch logType {
	case redfish.EVENT_TYPE_SYSTEM:
		return redfishApi.ClearSystemLogs(ctx)
	case redfish.EVENT_TYPE_MANAGER:
		return redfishApi.ClearManagerLogs(ctx)
	}
	return errors.Wrap(httperrors.ErrNotSupported, logType)
}

func (b *SBaremetalInstance) doCronJobs(ctx context.Context) {
	for _, job := range b.cronJobs {
		now := time.Now().UTC()
		if job.NeedsToRun(now) {
			// log.Debugf("need to run %s", job.Name())
			func() {
				job.StartRun()
				defer job.StopRun()
				err := job.Do(ctx, now)
				if err != nil {
					log.Errorf("Baremetal %s do cronjob %s fail: %s", b.GetName(), job.Name(), err)
				}
			}()
		}
	}
}

func (b *SBaremetalInstance) fetchPowerThermalMetrics(ctx context.Context) ([]influxdb.SKeyValue, []influxdb.SKeyValue, error) {
	redfishApi := b.GetRedfishCli(ctx)
	if redfishApi == nil {
		return nil, nil, errors.Wrap(httperrors.ErrNotSupported, "no valid redfish api")
	}
	powers, err := redfishApi.GetPower(ctx)
	if err != nil {
		return nil, nil, errors.Wrap(err, "redfishApi.GetPower")
	}
	thermals, err := redfishApi.GetThermal(ctx)
	if err != nil {
		return nil, nil, errors.Wrap(err, "redfishApi.Thermal")
	}

	var powerMetrics []influxdb.SKeyValue
	if len(powers) > 0 {
		powerMetrics = powers[0].ToMetrics()
	}
	thermalMetrics := make([]influxdb.SKeyValue, 0)
	for _, t := range thermals {
		thermalMetrics = append(thermalMetrics, t.ToMetric())
	}
	return powerMetrics, thermalMetrics, nil
}

func (b *SBaremetalInstance) GetConsoleJNLP(ctx context.Context) (string, error) {
	cli := b.GetRedfishCli(ctx)
	if cli != nil {
		return cli.GetConsoleJNLP(ctx)
	}
	conf := b.GetIPMIConfig()
	bmc := bmconsole.NewBMCConsole(conf.IpAddr, conf.Username, conf.Password, false)
	manufacture := b.GetManufacture()
	switch strings.ToLower(manufacture) {
	case "hp", "hpe":
		return bmc.GetIloConsoleJNLP(ctx)
	case "dell", "dell inc.":
		return bmc.GetIdracConsoleJNLP(ctx, b.GetSerialNumber(), b.GetModel())
	case "supermicro":
		return bmc.GetSupermicroConsoleJNLP(ctx)
	case "lenovo":
		return bmc.GetLenovoConsoleJNLP(ctx)
	}
	return "", httperrors.NewNotImplementedError("Unsupported manufacture %s", manufacture)
}

func (b *SBaremetalInstance) getTags() []influxdb.SKeyValue {
	tags := []influxdb.SKeyValue{
		{
			Key:   "id",
			Value: b.GetId(),
		},
		{
			Key:   "name",
			Value: b.GetName(),
		},
		{
			Key:   "zone_id",
			Value: b.GetZoneId(),
		},
		{
			Key:   "zone",
			Value: b.GetZoneName(),
		},
		{
			Key:   "boot_mode",
			Value: b.GetBootMode(),
		},
		{
			Key:   "host_type",
			Value: "baremetal",
		},
		{
			Key:   "region",
			Value: b.GetRegion(),
		},
		{
			Key:   "region_id",
			Value: b.GetRegionId(),
		},
		{
			Key:   "sn",
			Value: b.GetSerialNumber(),
		},
		{
			Key:   "manufacture",
			Value: b.GetManufacture(),
		},
		{
			Key:   "model",
			Value: b.GetModel(),
		},
		{
			Key:   "status",
			Value: b.GetStatus(),
		},
		{
			Key:   "ncpu",
			Value: b.GetNodeCount(),
		},
		{
			Key:   "mem_gb",
			Value: b.GetMemGb(),
		},
	}
	srvId := b.GetServerId()
	if len(srvId) > 0 {
		tags = append(tags, influxdb.SKeyValue{
			Key:   "server_id",
			Value: srvId,
		})
		tags = append(tags, influxdb.SKeyValue{
			Key:   "server",
			Value: b.GetServerName(),
		})
	}
	return tags
}

type SBaremetalServer struct {
	baremetal *SBaremetalInstance
	desc      *jsonutils.JSONDict
}

func newBaremetalServer(baremetal *SBaremetalInstance, desc *jsonutils.JSONDict) (*SBaremetalServer, error) {
	server := &SBaremetalServer{
		baremetal: baremetal,
		desc:      desc,
	}
	err := server.SaveDesc(desc)
	return server, err
}

func (server *SBaremetalServer) GetId() string {
	id, err := server.desc.GetString("uuid")
	if err != nil {
		log.Fatalf("Get id from desc error: %v", err)
	}
	return id
}

func (server *SBaremetalServer) GetName() string {
	id, err := server.desc.GetString("name")
	if err != nil {
		log.Errorf("Get name from desc %s error: %v", server.desc.String(), err)
	}
	return id
}

func (server *SBaremetalServer) GetHostName() string {
	hostname, err := server.desc.GetString("hostname")
	if err != nil {
		log.Errorf("Get hostname from desc %s error: %v", server.desc.String(), err)
	}
	return hostname
}

func (server *SBaremetalServer) GetOsName() string {
	osName, err := server.desc.GetString("os_name")
	if err != nil {
		log.Errorf("Get os name from desc %s error: %v", server.desc.String(), err)
	}
	return osName
}

func (server *SBaremetalServer) SaveDesc(desc jsonutils.JSONObject) error {
	if desc != nil {
		server.desc = desc.(*jsonutils.JSONDict)
	}
	return ioutil.WriteFile(server.baremetal.GetServerDescFilePath(), []byte(server.desc.String()), 0644)
}

func (s *SBaremetalServer) RemoveDesc() {
	os.Remove(s.baremetal.GetServerDescFilePath())
	s.desc = nil
	s.baremetal = nil
}

func (s *SBaremetalServer) GetRootTemplateId() string {
	rootDisk, err := s.desc.GetAt(0, "disks")
	if err != nil {
		log.Errorf("Can't found root disk")
		return ""
	}
	id, _ := rootDisk.GetString("template_id")
	return id
}

func (s *SBaremetalServer) GetMetadata() (*jsonutils.JSONDict, error) {
	metadata, err := s.desc.Get("metadata")
	if err != nil {
		return nil, errors.Wrap(err, "get desc.metadata")
	}
	return metadata.(*jsonutils.JSONDict), nil
}

func (s *SBaremetalServer) GetRootDiskMatcher() (*api.BaremetalRootDiskMatcher, error) {
	matcher, err := s.getRootDiskMatcher()
	if err != nil && errors.Cause(err) != errors.ErrNotFound {
		return nil, errors.Wrap(err, "getRootDiskMatcher")
	}
	if matcher == nil {
		matcher = &api.BaremetalRootDiskMatcher{}
	}
	rootDiskObj, err := s.GetRootDiskObj()
	if err != nil {
		return nil, errors.Wrap(err, "GetRootDiskObj")
	}
	pciPath, _ := rootDiskObj.GetString("pci_path")
	if pciPath != "" {
		matcher.PCIPath = pciPath
	}
	return matcher, nil
}

func (s *SBaremetalServer) getRootDiskMatcher() (*api.BaremetalRootDiskMatcher, error) {
	metadata, err := s.GetMetadata()
	if err != nil {
		return nil, errors.Wrap(err, "get metadata")
	}
	if !metadata.Contains(api.BAREMETAL_SERVER_METATA_ROOT_DISK_MATCHER) {
		return nil, errors.Wrapf(errors.ErrNotFound, "not found %s in metadata", api.BAREMETAL_SERVER_METATA_ROOT_DISK_MATCHER)
	}
	jStr, _ := metadata.GetString(api.BAREMETAL_SERVER_METATA_ROOT_DISK_MATCHER)
	jObj, err := jsonutils.ParseString(jStr)
	if err != nil {
		return nil, errors.Wrapf(err, "parse json string: %s", jStr)
	}
	matcher := new(api.BaremetalRootDiskMatcher)
	if err := jObj.Unmarshal(matcher); err != nil {
		return nil, errors.Wrapf(err, "unmarshal to matcher")
	}
	return matcher, nil
}

func (s *SBaremetalServer) GetDiskConfig() ([]*api.BaremetalDiskConfig, error) {
	layouts := make([]baremetal.Layout, 0)
	err := s.desc.Unmarshal(&layouts, "disk_config")
	if err != nil {
		return nil, err
	}

	if len(layouts) != 0 {
		firstDisk := layouts[0]
		// convert to normal order if first disk is Linux or PCIE driver
		driver := firstDisk.Conf.Driver
		if utils.IsInStringArray(driver, []string{baremetal.DISK_DRIVER_LINUX, baremetal.DISK_DRIVER_PCIE}) {
			return baremetal.GetLayoutDiskConfig(layouts), nil
		}
	}
	return baremetal.GetLayoutRaidConfig(layouts), nil
}

func (s *SBaremetalServer) GetRootDiskObj() (*jsonutils.JSONDict, error) {
	disks, _ := s.desc.GetArray("disks")
	if len(disks) == 0 {
		return nil, errors.Error("Empty disks in desc")
	}
	return disks[0].(*jsonutils.JSONDict), nil
}

func (s *SBaremetalServer) NewConfigedSSHPartitionTool(term *ssh.Client) (*disktool.SSHPartitionTool, error) {
	raid, nonRaid, pcie, err := detect_storages.DetectStorageInfo(term, false)
	if err != nil {
		return nil, errors.Wrap(err, "DetectStorageInfo")
	}
	storages := make([]*baremetal.BaremetalStorage, 0)
	storages = append(storages, raid...)
	storages = append(storages, nonRaid...)
	storages = append(storages, pcie...)
	confs, err := s.GetDiskConfig()
	if err != nil {
		return nil, err
	}
	layouts, err := baremetal.CalculateLayout(confs, storages)
	if err != nil {
		return nil, fmt.Errorf("CalculateLayout: %v", err)
	}

	log.Errorf("NewConfigedSSHPartitionTool layouts: %s", jsonutils.Marshal(layouts))
	diskConfs := baremetal.GroupLayoutResultsByDriverAdapter(layouts)
	for _, dConf := range diskConfs {
		driver := dConf.Driver
		adapter := dConf.Adapter
		isSoftRaid := baremetal.DISK_DRIVERS_SOFT_RAID.Has(driver)

		raidDrv := raiddrivers.GetDriver(driver, term)
		if raidDrv != nil {
			if err := raidDrv.ParsePhyDevs(); err != nil {
				return nil, fmt.Errorf("RaidDriver %s parse physical devices: %v", raidDrv.GetName(), err)
			}
			if isSoftRaid {
				devs := make([]*baremetal.BaremetalStorage, 0)
				for _, layout := range layouts {
					if len(layout.Disks) > 0 && layout.Disks[0].Driver == driver && layout.Disks[0].Adapter == dConf.Adapter {
						devs = append(devs, layout.Disks...)
					}
				}

				log.Infof("SetDevicesForAdapter %v", jsonutils.Marshal(devs))
				if mdadmDrver, ok := raidDrv.(raid2.IRaidDeviceSetter); ok {
					mdadmDrver.SetDevicesForAdapter(dConf.Adapter, devs)
				}
			}

			if err := raiddrivers.PostBuildRaid(raidDrv, adapter); err != nil {
				return nil, fmt.Errorf("Build %s raid failed: %v", raidDrv.GetName(), err)
			}
			time.Sleep(10 * time.Second) // wait 10 seconds for raid status OK
		}
	}

	matcher, err := s.GetRootDiskMatcher()
	if errors.Cause(err) != errors.ErrNotFound {
		log.Errorf("GetRootDiskMatcher: %v", err)
	}
	tool, err := disktool.NewSSHPartitionTool(term, layouts, matcher)
	if err != nil {
		return nil, errors.Wrap(err, "NewSSHPartitionTool")
	}
	return tool, nil
}

func (s *SBaremetalServer) DoDiskConfig(term *ssh.Client) (*disktool.SSHPartitionTool, error) {
	raid, nonRaid, pcie, err := detect_storages.DetectStorageInfo(term, true)
	if err != nil {
		return nil, errors.Wrap(err, "DetectStorageInfo")
	}
	storages := make([]*baremetal.BaremetalStorage, 0)
	storages = append(storages, raid...)
	storages = append(storages, nonRaid...)
	storages = append(storages, pcie...)
	confs, err := s.GetDiskConfig()
	if err != nil {
		return nil, err
	}
	layouts, err := baremetal.CalculateLayout(confs, storages)
	if err != nil {
		return nil, fmt.Errorf("CalculateLayout: %v", err)
	}
	diskConfs := baremetal.GroupLayoutResultsByDriverAdapter(layouts)
	log.Errorf("%s layouts: %s, diskConfs: %s", s.GetName(), jsonutils.Marshal(layouts).PrettyString(), jsonutils.Marshal(diskConfs).PrettyString())
	for _, dConf := range diskConfs {
		driver := dConf.Driver
		raidDrv := raiddrivers.GetDriver(driver, term)
		if raidDrv != nil {
			if err := raidDrv.ParsePhyDevs(); err != nil {
				return nil, fmt.Errorf("RaidDriver %s parse physical devices: %v", raidDrv.GetName(), err)
			}
			raidDrv.CleanRaid()
		}
	}

	for _, dConf := range diskConfs {
		driver := dConf.Driver
		adapter := dConf.Adapter
		isSoftRaid := baremetal.DISK_DRIVERS_SOFT_RAID.Has(driver)

		raidDrv := raiddrivers.GetDriver(driver, term)
		if raidDrv != nil {
			if err := raidDrv.ParsePhyDevs(); err != nil {
				return nil, fmt.Errorf("RaidDriver %s parse physical devices: %v", raidDrv.GetName(), err)
			}
			if isSoftRaid {
				devs := make([]*baremetal.BaremetalStorage, 0)
				for _, layout := range layouts {
					if len(layout.Disks) > 0 && layout.Disks[0].Driver == driver && layout.Disks[0].Adapter == dConf.Adapter {
						devs = append(devs, layout.Disks...)
					}
				}

				log.Infof("SetDevicesForAdapter %v", jsonutils.Marshal(devs))
				if mdadmDriver, ok := raidDrv.(raid2.IRaidDeviceSetter); ok {
					mdadmDriver.SetDevicesForAdapter(dConf.Adapter, devs)
				}
			}

			if err := raiddrivers.BuildRaid(raidDrv, dConf.Configs, adapter); err != nil {
				return nil, fmt.Errorf("Build %s raid failed: %v", raidDrv.GetName(), err)
			}
			time.Sleep(10 * time.Second) // wait 10 seconds for raid status OK
		}
	}

	matcher, err := s.GetRootDiskMatcher()
	if errors.Cause(err) != errors.ErrNotFound {
		log.Errorf("GetRootDiskMatcher: %v", err)
	}
	tool, err := disktool.NewSSHPartitionTool(term, layouts, matcher)
	if err != nil {
		return nil, errors.Wrap(err, "NewSSHPartitionTool")
	}
	maxTries := 60
	for tried := 0; !tool.IsAllDisksReady() && tried < maxTries; tried++ {
		time.Sleep(5 * time.Second)
		tool.RetrieveDiskInfo(matcher)
		log.Warningf("disktool not ready string: %s", tool.DebugString())
	}

	if !tool.IsAllDisksReady() {
		return nil, fmt.Errorf("Raid disks are not ready???")
	}

	return tool, nil
}

func (s *SBaremetalServer) DoDiskUnconfig(term *ssh.Client) error {
	// tear down raid
	driver := s.baremetal.GetStorageDriver()
	raidDrv := raiddrivers.GetDriver(driver, term)
	if raidDrv != nil {
		if err := raidDrv.ParsePhyDevs(); err != nil {
			return err
		}
		raidDrv.CleanRaid()
	}
	return nil
}

func (s *SBaremetalServer) DoEraseDisk(term *ssh.Client) error {
	// soft raid should stop mdadm first
	if err := mdadm.CleanRaid(term); err != nil {
		return err
	}
	cmd := "/lib/mos/partdestroy.sh"
	_, err := term.Run(cmd)
	return err
}

func replaceHostAddr(urlStr string, addr string) string {
	urlComp, _ := url.Parse(urlStr)
	commaPos := strings.IndexByte(urlComp.Host, ':')
	if commaPos >= 0 {
		urlComp.Host = addr + urlComp.Host[commaPos:]
	} else {
		urlComp.Host = addr
	}
	return urlComp.String()
}

func (s *SBaremetalServer) doCreateRoot(term *ssh.Client, devName string, disableImageCache bool) error {
	session := s.baremetal.GetClientSession()
	token := session.GetToken().GetTokenString()
	urlStr := s.baremetal.GetImageUrl(disableImageCache)
	imageId := s.GetRootTemplateId()
	cmd := fmt.Sprintf("/lib/mos/rootcreate.sh %s %s %s %s", token, urlStr, imageId, devName)
	log.Infof("rootcreate cmd: %q", cmd)
	if _, err := term.Run(cmd); err != nil {
		return fmt.Errorf("Root create fail: %v", err)
	}
	return nil
}

func (s *SBaremetalServer) DoPartitionDisk(tool *disktool.SSHPartitionTool, term *ssh.Client, disableImageCache bool) (*disktool.DiskPartitions, []*disktool.Partition, error) {
	raid, nonRaid, pcie, err := detect_storages.DetectStorageInfo(term, false)
	if err != nil {
		return nil, nil, errors.Wrap(err, "DetectStorageInfo")
	}
	storages := make([]*baremetal.BaremetalStorage, 0)
	storages = append(storages, raid...)
	storages = append(storages, nonRaid...)
	storages = append(storages, pcie...)
	// confs, err := s.GetDiskConfig()
	// if err != nil {
	// 	return nil, errors.Wrapf(err, "do disk config")
	// }
	// layouts, err := baremetal.CalculateLayout(confs, storages)
	// if err != nil {
	// 	return nil, errors.Wrapf(err, "CalculateLayout")
	// }
	//
	// tool, err := disktool.NewSSHPartitionTool(term, layouts)
	// if err != nil {
	// 	return nil, errors.Wrapf(err, "NewSSHPartitionTool")
	// }

	disks, _ := s.desc.GetArray("disks")
	if len(disks) == 0 {
		return nil, nil, errors.Error("Empty disks in desc")
	}

	rootImageId := s.GetRootTemplateId()
	diskOffset := 0
	rootDisk := tool.GetRootDisk()
	log.Infof("root disk name %s", rootDisk.GetDevName())
	if len(rootImageId) > 0 {
		rootDiskObj := disks[0]
		rootSize, _ := rootDiskObj.Int("size")
		err = s.doCreateRoot(term, rootDisk.GetDevName(), disableImageCache)
		if err != nil {
			return rootDisk, nil, errors.Wrap(err, "Failed to create root")
		}
		tool.RetrievePartitionInfo()
		parts := tool.GetPartitions()
		if len(parts) == 0 {
			return rootDisk, nil, errors.Error("Root disk create failed, no partitions")
		}
		log.Infof("Resize root to %d MB", rootSize)
		if err := tool.ResizePartition(0, rootSize); err != nil {
			return rootDisk, nil, errors.Wrapf(err, "Fail to resize root to %d", rootSize)
		}
		diskOffset = 1
	} else {
		tool.RetrievePartitionInfo()
		parts := tool.GetPartitions()
		if len(parts) > 0 {
			return rootDisk, nil, errors.Error("should no partition!!!")
		}
	}

	if len(disks) > diskOffset {
		for _, disk := range disks[diskOffset:] {
			sz, err := disk.Int("size")
			if err != nil {
				sz = -1
			}
			fs, _ := disk.GetString("fs")
			uuid, _ := disk.GetString("disk_id")
			driver, _ := disk.GetString("driver")
			log.Infof("Create partition %d %s", sz, fs)
			if err := tool.CreatePartition(-1, sz, fs, true, driver, uuid); err != nil {
				return rootDisk, nil, errors.Wrapf(err, "Fail to create disk %s", disk.String())
			}
		}
	}
	log.Infof("Finish create partitions")

	return rootDisk, tool.GetPartitions(), nil
}

func (s *SBaremetalServer) DoRebuildRootDisk(tool *disktool.SSHPartitionTool, term *ssh.Client, disableImageCache bool) (*disktool.DiskPartitions, []*disktool.Partition, error) {
	// raid, nonRaid, pcie, err := detect_storages.DetectStorageInfo(term, false)
	// if err != nil {
	// 	return nil, err
	// }
	// storages := make([]*baremetal.BaremetalStorage, 0)
	// storages = append(storages, raid...)
	// storages = append(storages, nonRaid...)
	// storages = append(storages, pcie...)
	// confs, err := s.GetDiskConfig()
	// if err != nil {
	// 	return nil, err
	// }
	// layouts, err := baremetal.CalculateLayout(confs, storages)
	// if err != nil {
	// 	return nil, err
	// }
	//
	// tool, err := disktool.NewSSHPartitionTool(term, layouts)
	// if err != nil {
	// 	return nil, err
	// }

	disks, _ := s.desc.GetArray("disks")
	if len(disks) == 0 {
		return nil, nil, fmt.Errorf("Empty disks in desc")
	}

	rootDisk := disks[0]
	rootSize, _ := rootDisk.Int("size")
	rd := tool.GetRootDisk()
	err := s.doCreateRoot(term, rd.GetDevName(), disableImageCache)
	if err != nil {
		return rd, nil, fmt.Errorf("Failed to create root: %v", err)
	}
	tool.RetrievePartitionInfo()
	if err := rd.ReInitInfo(); err != nil {
		return rd, nil, errors.Wrap(err, "Reinit root disk after create root")
	}

	log.Infof("Resize root to %d MB", rootSize)
	if err := rd.ResizePartition(rootSize); err != nil {
		return rd, nil, fmt.Errorf("Fail to resize root to %d, err: %v", rootSize, err)
	}
	if len(disks) > 1 {
		for _, disk := range disks[1:] {
			sz, err := disk.Int("size")
			if err != nil {
				sz = -1
			}
			fs, _ := disk.GetString("fs")
			uuid, _ := disk.GetString("disk_id")
			log.Infof("Create partition %d %s", sz, fs)
			if err := rd.CreatePartition(sz, fs, false, uuid); err != nil {
				log.Errorf("Rebuild root create (%s, %d, %s) partition error: %v", uuid, sz, fs, err)
				break
			}
		}
	}
	log.Infof("Finish create partitions")

	parts := rd.GetPartitions()
	restDisks := tool.Disks()
	if len(restDisks) > 1 {
		restDisks = restDisks[1:]
	}
	for _, d := range restDisks {
		parts = append(parts, d.GetPartitions()...)
	}
	return rd, parts, nil
}

func (s *SBaremetalServer) SyncPartitionSize(term *ssh.Client, rootDisk *disktool.DiskPartitions, parts []*disktool.Partition) ([]jsonutils.JSONObject, error) {
	disks, _ := s.desc.GetArray("disks")

	// calculate root partitions count
	rootPartsCnt := len(parts) - len(disks) + 1
	if len(parts) < len(disks) {
		// HACK: rebuild root disk
		rootPartsCnt = len(parts)
	}
	rootParts := parts[0:rootPartsCnt]
	dataParts := parts[rootPartsCnt:]
	idx := 0

	// set root disk attributes that returns to region service
	size := (rootParts[len(rootParts)-1].GetEnd() + 1) * 512 / 1024 / 1024
	rootDiskObj := disks[idx].(*jsonutils.JSONDict)
	rootDiskObj.Set("size", jsonutils.NewInt(int64(size)))
	rootDiskObj.Set("pci_path", jsonutils.NewString(rootDisk.GetPCIPath()))

	idx += 1
	for _, p := range dataParts {
		sizeMB, err := p.GetSizeMB()
		if err != nil {
			return nil, errors.Wrap(err, "GetSizeMB")
		}
		disks[idx].(*jsonutils.JSONDict).Set("size", jsonutils.NewInt(int64(sizeMB)))
		disks[idx].(*jsonutils.JSONDict).Set("dev", jsonutils.NewString(p.GetDev()))
		idx++
	}
	s.desc.Set("disks", jsonutils.NewArray(disks...))
	return disks, nil
}

func (s *SBaremetalServer) DoDeploy(tool *disktool.SSHPartitionTool, term *ssh.Client, data jsonutils.JSONObject, isInit bool) (jsonutils.JSONObject, error) {
	publicKey := deployapi.GetKeys(data)
	isRandomPassword := false
	password, _ := data.GetString("password")
	resetPassword := jsonutils.QueryBoolean(data, "reset_password", false)
	if resetPassword && len(password) == 0 {
		password = seclib.RandomPassword(12)
		isRandomPassword = true
	}
	deployArray := make([]*deployapi.DeployContent, 0)
	if data.Contains("deploys") {
		err := data.Unmarshal(&deployArray, "deploys")
		if err != nil {
			return nil, errors.Wrapf(err, "unmarshal to array of deployapi.DeployContent")
		}
	}
	userData, _ := s.desc.GetString("user_data")
	deployInfo := deployapi.NewDeployInfo(publicKey, deployArray,
		password, isRandomPassword, isInit, true, o.Options.LinuxDefaultRootUser, o.Options.WindowsDefaultAdminUser, false, "",
		false, "",
		userData,
	)
	return s.deployFs(tool, term, deployInfo)
}

func (s *SBaremetalServer) deployFs(tool *disktool.SSHPartitionTool, term *ssh.Client, deployInfo *deployapi.DeployInfo) (jsonutils.JSONObject, error) {
	raid, nonRaid, pcie, err := detect_storages.DetectStorageInfo(term, false)
	if err != nil {
		return nil, err
	}
	storages := make([]*baremetal.BaremetalStorage, 0)
	storages = append(storages, raid...)
	storages = append(storages, nonRaid...)
	storages = append(storages, pcie...)
	confs, err := s.GetDiskConfig()
	if err != nil {
		return nil, err
	}
	layouts, err := baremetal.CalculateLayout(confs, storages)
	if err != nil {
		return nil, err
	}
	rootDev, rootfs, err := sshpart.MountSSHRootfs(tool, term, layouts)
	if err != nil {
		return nil, fmt.Errorf("Find rootfs error: %s", err)
	}
	defer func() {
		rootDev.Umount()
	}()
	if strings.ToLower(rootfs.GetOs()) == "windows" {
		return nil, fmt.Errorf("Unsupported OS: %s", rootfs.GetOs())
	}
	deployDesc, err := deployapi.GuestJsonDescToDeployDesc(s.desc)
	if err != nil {
		return nil, errors.Wrap(err, "To deploy desc fail")
	}
	deployDesc.Hypervisor = api.HYPERVISOR_BAREMETAL
	deployDesc, err = s.reIndexDescNics(term, deployDesc)
	if err != nil {
		return nil, errors.Wrap(err, "reIndexDescNics")
	}

	return guestfs.DeployGuestFs(rootfs, deployDesc, deployInfo)
}

func (s *SBaremetalServer) reIndexDescNics(term *ssh.Client, desc *deployapi.GuestDesc) (*deployapi.GuestDesc, error) {
	rNics, err := tasks.GetNicsInfo(term)
	if err != nil {
		return nil, errors.Wrapf(err, "Get server %s remote nics", s.GetName())
	}
	findRemoteNic := func(mac string) (int, *types.SNicDevInfo) {
		for idx, nic := range rNics {
			if nic.Mac.String() == mac {
				return idx, rNics[idx]
			}
		}
		return -1, nil
	}
	reIndexNics := func(nics []*deployapi.Nic) ([]*deployapi.Nic, error) {
		for idx, nic := range nics {
			if nic.GetNicType() == string(api.NIC_TYPE_IPMI) || nic.GetIndex() >= 0 {
				continue
			}
			rIdx, rNic := findRemoteNic(nic.GetMac())
			if rNic == nil {
				return nil, errors.Errorf("Not found remote nic by mac %q", nic.GetMac())
			}
			if nics[idx].Index != int32(rIdx) {
				log.Infof("Reindex nic %s index %d to %d", nics[idx].GetMac(), nics[idx].Index, rIdx)
				nics[idx].Index = int32(rIdx)
			}
		}
		return nics, nil
	}
	nics, err := reIndexNics(desc.Nics)
	if err != nil {
		return nil, errors.Wrap(err, "Reindex nics")
	}
	desc.Nics = nics
	sNics, err := reIndexNics(desc.NicsStandby)
	if err != nil {
		return nil, errors.Wrap(err, "Reindex standby nics")
	}
	desc.NicsStandby = sNics
	return desc, nil
}

func (s *SBaremetalServer) GetNics() []types.SServerNic {
	nics := []types.SServerNic{}
	err := s.desc.Unmarshal(&nics, "nics")
	if err != nil {
		log.Errorf("Unmarshal desc to get server nics error: %v", err)
		return nil
	}
	return nics
}

func (s *SBaremetalServer) GetNicByMac(mac net.HardwareAddr) *types.SNic {
	for _, n := range s.GetNics() {
		if n.GetMac().String() == mac.String() {
			nic := n.ToNic()
			return &nic
		}
	}
	return nil
}