Cloudpods/backend/pkg/hostman/diskutils/vddk.go

// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package diskutils

import (
	"bytes"
	"crypto/sha1"
	"crypto/tls"
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"net"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"time"

	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"

	"yunion.io/x/onecloud/pkg/hostman/guestfs"
	"yunion.io/x/onecloud/pkg/hostman/guestfs/fsdriver"
	"yunion.io/x/onecloud/pkg/hostman/guestfs/kvmpart"
	"yunion.io/x/onecloud/pkg/hostman/hostdeployer/apis"
	"yunion.io/x/onecloud/pkg/util/qemuimg"
)

const (
	TMPDIR = "/tmp/vmware-root"
)

var (
	MNT_PATTERN = regexp.MustCompile(`Disk flat file mounted under ([^\s]+)`)
)

type VDDKDisk struct {
	Host     string
	Port     int
	User     string
	Passwd   string
	VmRef    string
	DiskPath string

	FUseDir  string
	PartDirs []string
	Proc     *Command
	Pid      int

	kvmDisk      *SKVMGuestDisk
	readOnly     bool
	deployDriver string
}

func NewVDDKDisk(vddkInfo *apis.VDDKConInfo, diskPath, deployDriver string, readOnly bool) (*VDDKDisk, error) {
	return &VDDKDisk{
		Host:         vddkInfo.Host,
		Port:         int(vddkInfo.Port),
		User:         vddkInfo.User,
		Passwd:       vddkInfo.Passwd,
		VmRef:        vddkInfo.Vmref,
		DiskPath:     diskPath,
		deployDriver: deployDriver,
		readOnly:     readOnly,
	}, nil
}

type Command struct {
	*exec.Cmd
	done      chan error
	stdouterr *bytes.Buffer
	stdin     io.Writer
}

func NewCommand(name string, arg ...string) *Command {
	cmd := Command{
		Cmd:  exec.Command(name, arg...),
		done: make(chan error, 1),
	}
	cmd.stdouterr = bytes.NewBuffer([]byte{})
	cmd.Stdout = cmd.stdouterr
	cmd.Stderr = cmd.stdouterr
	cmd.stdin, _ = cmd.StdinPipe()
	return &cmd
}

func (c *Command) Send(msg []byte) error {
	_, err := c.stdin.Write(msg)
	return err
}

func (c *Command) Start() error {
	if err := c.Cmd.Start(); err != nil {
		return err
	}
	go func() {
		c.done <- c.Cmd.Wait()
	}()
	return nil
}

func (c *Command) Exited() bool {
	return len(c.done) == 1
}

// Wait will block
func (c *Command) Wait() error {
	return <-c.done
}

func (c *Command) Kill() error {
	return c.Process.Kill()
}

func execpath() string {
	return "/opt/vmware-vddk/bin/vix-mntapi-sample"
}

func libdir() string {
	return "/usr/lib/vmware"
}

func logpath(pid int) string {
	return fmt.Sprintf("%s/vixDiskLib-%d.log", TMPDIR, pid)
}

func (vd *VDDKDisk) Cleanup() {
	if vd.kvmDisk != nil {
		vd.kvmDisk.Cleanup()
		vd.kvmDisk = nil
	}
}

func (vd *VDDKDisk) Connect(*apis.GuestDesc) error {
	flatFile, err := vd.ConnectBlockDevice()
	if err != nil {
		return errors.Wrap(err, "ConnectBlockDevice")
	}
	vd.kvmDisk, err = NewKVMGuestDisk(qemuimg.SImageInfo{Path: flatFile}, vd.deployDriver, vd.readOnly)
	if err != nil {
		vd.DisconnectBlockDevice()
		return errors.Wrap(err, "NewKVMGuestDisk")
	}
	err = vd.kvmDisk.Connect(nil)
	if err != nil {
		vd.DisconnectBlockDevice()
		return errors.Wrap(err, "kvmDisk connect")
	}
	return nil
}

func (vd *VDDKDisk) ConnectWithDiskId(desc *apis.GuestDesc, diskId string) error {
	return vd.Connect(desc)
}

func (vd *VDDKDisk) Disconnect() error {
	if vd.kvmDisk != nil {
		if err := vd.kvmDisk.Disconnect(); err != nil {
			log.Errorf("kvm disk disconnect failed %s", err)
		}
		vd.kvmDisk.Cleanup()
		vd.kvmDisk = nil
	}
	return vd.DisconnectBlockDevice()
}

func (vd *VDDKDisk) MountRootfs() (fsdriver.IRootFsDriver, error) {
	if vd.kvmDisk == nil {
		return nil, fmt.Errorf("kvmDisk is nil")
	}
	return vd.kvmDisk.MountRootfs()
}

func (vd *VDDKDisk) UmountRootfs(fd fsdriver.IRootFsDriver) error {
	if vd.kvmDisk == nil {
		return nil
	}
	return vd.kvmDisk.UmountRootfs(fd)
}

func (vd *VDDKDisk) ParsePartitions(buf string) error {
	// Disk flat file mounted under /run/vmware/fuse/7673253059900458465
	// Mounted Volume 1, Type 1, isMounted 1, symLink /tmp/vmware-root/7673253059900458465_1, numGuestMountPoints 0 (<null>)
	// print buf
	ms := MNT_PATTERN.FindAllStringSubmatch(buf, -1)
	if len(ms) != 0 {
		vd.FUseDir = ms[0][1]
		diskId := filepath.Base(vd.FUseDir)
		files, err := ioutil.ReadDir(TMPDIR)
		if err != nil {
			return errors.Wrapf(err, "ioutil.ReadDir for %s", TMPDIR)
		}
		for _, f := range files {
			if strings.HasPrefix(f.Name(), diskId) {
				vd.PartDirs = append(vd.PartDirs, filepath.Join(TMPDIR, f.Name()))
			}
		}
	}
	log.Infof("Fuse path: %s partitiaons: %s", vd.FUseDir, vd.PartDirs)
	return nil
}

func (vd *VDDKDisk) Mount() (err error) {
	defer func() {
		if err == nil {
			return
		}
		vd.Proc = nil
		log.Errorf("Exec vix-mntapi-sample error: %s", err)
	}()

	err = vd.ExecProg()
	if err != nil {
		return errors.Wrap(err, "VDDKDisk.ExecProg")
	}
	err = vd.WaitMounted()
	if err != nil {
		return errors.Wrap(err, "VDDKDisk.Mount")
	}
	return nil
}

func (vd *VDDKDisk) Umount() error {
	if vd.Proc != nil {
		err := vd.Proc.Send([]byte{'y'})
		if err != nil {
			errors.Wrap(err, "send 'y' to VDDKDisk.Proc")
		}
		err = vd.Proc.Wait()
		if err != nil {
			return errors.Wrap(err, "vd.Proc.Wait")
		}
	}
	if len(vd.FUseDir) != 0 {
		for _, p := range append(vd.PartDirs, vd.FUseDir) {
			vd.fuseUmount(p)
		}
	}
	if vd.Pid != 0 {
		logpath := logpath(vd.Pid)
		_, err := os.Stat(logpath)
		if err == nil || os.IsExist(err) {
			os.Remove(logpath)
		}
	}
	return nil
}

func (vd *VDDKDisk) fuseUmount(path string) {
	maxTries, tried := 4, 0

	_, err := os.Stat(path)
	if err != nil && os.IsNotExist(err) {
		// no such path
		return
	}

	for tried < maxTries {
		tried += 1
		err := exec.Command("umount", path).Run()
		if err != nil {
			time.Sleep(time.Duration(tried) * 15 * time.Second)
			log.Errorf("Fail to umount %s: %s", path, err)
			continue
		}
		_, err = os.Stat(path)
		if err == nil || os.IsExist(err) {
			err = exec.Command("rm", "-rf", path).Run()
			if err != nil {
				time.Sleep(time.Duration(tried) * 15 * time.Second)
				log.Errorf("Fail to umount %s: %s", path, err)
				continue
			}
		}
	}
}

func (vd *VDDKDisk) ExecProg() error {
	thumb, err := vd.getServerCertThumbSha1(fmt.Sprintf("%s:%d", vd.Host, vd.Port))
	if err != nil {
		return errors.Wrapf(err, "Fail contact server %s", vd.Host)
	}
	cmd := NewCommand(execpath(), "-info", "-host", vd.Host, "-port", strconv.Itoa(vd.Port), "-user", vd.User,
		"-password", vd.Passwd, "-mode", "nbd", "-thumb", thumb, "-vm", fmt.Sprintf("moref=%s", vd.VmRef), vd.DiskPath)
	log.Debugf("command to mount: %s", cmd)
	env := os.Environ()
	env = append(env, fmt.Sprintf("LD_LIBRARY_PATH=%s", libdir()))
	cmd.Env = env
	vd.Proc = cmd
	err = vd.Proc.Start()
	if err != nil {
		return errors.Wrap(err, "vd.Proc.Start")
	}
	vd.Pid = cmd.Process.Pid
	return nil
}

// getServerCertBin try to obtain the remote ssl certificate
func (vd *VDDKDisk) getServerCertBin(addr string) ([]byte, error) {
	rawConn, err := net.Dial("tcp", addr)
	if err != nil {
		return nil, errors.Wrapf(err, "net.Dial for addr '%s'", addr)
	}
	defer rawConn.Close()

	// get the wrpped conn
	sslWrappedConn := tls.Client(rawConn, &tls.Config{InsecureSkipVerify: true})
	err = sslWrappedConn.Handshake()
	if err != nil {
		return nil, errors.Wrapf(err, "fail to complete ssl handshake with addr '%s'", addr)
	}
	return sslWrappedConn.ConnectionState().PeerCertificates[0].Raw, nil
}

func (vd *VDDKDisk) getServerCertThumbSha1(addr string) (string, error) {
	certBin, err := vd.getServerCertBin(addr)
	if err != nil {
		return "", err
	}
	sha := sha1.Sum(certBin)
	shaHex := hex.EncodeToString(sha[:])
	length := len(shaHex) / 2 * 2
	tmp := make([][]byte, 0, length)
	for i := 1; i < length; i += 2 {
		tmp = append(tmp, []byte{shaHex[i-1], shaHex[i]})
	}
	return string(bytes.Join(tmp, []byte{':'})), nil
}

func (vd *VDDKDisk) WaitMounted() error {
	endStr := []byte("Do you want to procede to unmount the volume")
	timeout := 300 * time.Second
	endClock := time.After(timeout)
	isEnd := false

Loop:
	for !vd.Proc.Exited() {
		select {
		case <-endClock:
			break Loop
		default:
			if bytes.Contains(vd.Proc.stdouterr.Bytes(), endStr) {
				log.Debugf("find the mark")
				isEnd = true
				break Loop
			}
		}
		// Reduce inspection density
		time.Sleep(100 * time.Millisecond)
	}

	backup := vd.Proc.stdouterr.String()
	log.Debugf("%s", backup)
	err := vd.ParsePartitions(backup)
	if err != nil {
		return errors.Wrap(err, "VDDKDisk.ParsePartitions")
	}
	if vd.Proc.Exited() {
		retCode := vd.Proc.ProcessState.ExitCode()
		err := vd.Proc.Kill()
		if err != nil {
			log.Errorf("unable to kill process '%d'", vd.Proc.Process.Pid)
		}
		return errors.Error(fmt.Sprintf("VDDKDisk prog exit error(%d): %s", retCode, backup))
	} else if !isEnd {
		err := vd.Proc.Kill()
		if err != nil {
			log.Errorf("unable to kill process '%d'", vd.Proc.Process.Pid)
		}
		return errors.Error("VDDKDisk read timeout, program blocked")
	}
	return nil
}

// connect vddk disk as fuse block device on local host
// return fuse device path, is null error
func (vd *VDDKDisk) ConnectBlockDevice() (string, error) {
	thumb, err := vd.getServerCertThumbSha1(fmt.Sprintf("%s:%d", vd.Host, vd.Port))
	if err != nil {
		return "", errors.Wrapf(err, "Fail contact server %s", vd.Host)
	}
	cmd := NewCommand(execpath(), "-info", "-connect-disk", "-host", vd.Host, "-port", strconv.Itoa(vd.Port), "-user", vd.User,
		"-password", vd.Passwd, "-mode", "nbd", "-thumb", thumb, "-vm", fmt.Sprintf("moref=%s", vd.VmRef), vd.DiskPath)
	log.Infof("command to mount: %s", cmd)
	env := os.Environ()
	env = append(env, fmt.Sprintf("LD_LIBRARY_PATH=%s", libdir()))
	cmd.Env = env
	vd.Proc = cmd
	err = vd.Proc.Start()
	if err != nil {
		return "", errors.Wrap(err, "vd.Proc.Start")
	}
	vd.Pid = cmd.Process.Pid

	var (
		timeout  = time.After(30 * time.Second)
		matchStr = "Log: Disk flat file mounted under"
		flatFile string
	)
Loop:
	for !vd.Proc.Exited() {
		select {
		case <-timeout:
			break Loop
		default:
			if idx := strings.Index(vd.Proc.stdouterr.String(), matchStr); idx >= 0 {
				output := vd.Proc.stdouterr.String()
				output = output[idx+len(matchStr):]
				if idx := strings.Index(output, "\n"); idx < 0 {
					return "", fmt.Errorf("find disk flat file failed")
				} else {
					flatFile = strings.TrimSpace(output[:idx])
				}
				log.Infof("disk flat file mounted under %s", flatFile)
				break Loop
			}
		}
	}
	if vd.Proc.Exited() {
		log.Errorf("process is exited: %s", vd.Proc.stdouterr.String())
		return "", vd.Proc.Wait()
	}
	vd.FUseDir = flatFile
	return path.Join(flatFile, "flat"), nil
}

func (vd *VDDKDisk) DisconnectBlockDevice() error {
	if vd.Proc != nil {
		_, err := vd.Proc.stdin.Write([]byte("y\n"))
		if err != nil {
			return errors.Wrap(err, "send 'y' to VDDKDisk.Proc")
		}
		return vd.Umount()
	}
	return fmt.Errorf("vddk disk has not connected")
}

func (vd *VDDKDisk) DeployGuestfs(req *apis.DeployParams) (res *apis.DeployGuestFsResponse, err error) {
	return vd.kvmDisk.DeployGuestfs(req)
}

func (d *VDDKDisk) ResizeFs(req *apis.ResizeFsParams) (*apis.Empty, error) {
	return d.kvmDisk.ResizeFs(req)
}

func (d *VDDKDisk) FormatFs(req *apis.FormatFsParams) (*apis.Empty, error) {
	return d.kvmDisk.FormatFs(req)
}

func (d *VDDKDisk) SaveToGlance(req *apis.SaveToGlanceParams) (*apis.SaveToGlanceResponse, error) {
	return d.kvmDisk.SaveToGlance(req)
}

func (d *VDDKDisk) ProbeImageInfo(req *apis.ProbeImageInfoPramas) (*apis.ImageInfo, error) {
	return d.kvmDisk.ProbeImageInfo(req)
}

type VDDKPartition struct {
	*kvmpart.SLocalGuestFS
}

func (vp *VDDKPartition) Mount() bool {
	log.Warningf("VDDKPartition.Mount not implement")
	return true
}

func (vp *VDDKPartition) MountPartReadOnly() bool {
	log.Warningf("VDDKPartition.MountPartReadOnly not implement")
	return true
}

func (vp *VDDKPartition) Umount() error {
	log.Warningf("VDDKPartition.Umount not implement")
	return nil
}

func (vp *VDDKPartition) IsReadonly() bool {
	return guestfs.IsPartitionReadonly(vp)
}

func (vp *VDDKPartition) GetPhysicalPartitionType() string {
	log.Warningf("VDDKPartition.GetPhysicalPartitionType not implement")
	return ""
}

func (vp *VDDKPartition) GetPartDev() string {
	return ""
}

func (vp *VDDKPartition) IsMounted() bool {
	return true
}

func (vp *VDDKPartition) Zerofree() {
	return
}