Cloudpods/backend/pkg/hostman/container/storage/rbd/rbd.go

// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package rbd

import (
	"fmt"
	"path/filepath"
	"strings"

	"yunion.io/x/jsonutils"
	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"

	"yunion.io/x/onecloud/pkg/hostman/container/storage"
	"yunion.io/x/onecloud/pkg/util/fileutils2"
	"yunion.io/x/onecloud/pkg/util/procutils"
)

func init() {
	storage.RegisterDriver(newRbd())
}

type rbd struct{}

func newRbd() *rbd {
	return &rbd{}
}

func (r rbd) GetType() storage.StorageType {
	return storage.STORAGE_TYPE_RBD
}

func parseRbdPath(diskPath string) (pool, image, confPath, keyringPath string, err error) {
	if !strings.HasPrefix(diskPath, "rbd:") {
		return "", "", "", "", errors.Errorf("invalid rbd path: %s", diskPath)
	}
	diskPath = strings.TrimPrefix(diskPath, "rbd:")
	parts := strings.SplitN(diskPath, ":", 2)
	if len(parts) < 1 || parts[0] == "" {
		return "", "", "", "", errors.Errorf("invalid rbd path: missing pool/image")
	}
	poolImage := parts[0]
	slash := strings.Index(poolImage, "/")
	if slash <= 0 {
		return "", "", "", "", errors.Errorf("invalid rbd path: missing pool/image in %s", poolImage)
	}
	pool = poolImage[:slash]
	image = poolImage[slash+1:]
	if pool == "" || image == "" {
		return "", "", "", "", errors.Errorf("invalid rbd path: empty pool or image")
	}
	confPath = ""
	if len(parts) == 2 && parts[1] != "" {
		confPrefix := "conf="
		if strings.HasPrefix(parts[1], confPrefix) {
			confPath = strings.TrimPrefix(parts[1], confPrefix)
		}
	}
	if confPath == "" {
		return "", "", "", "", errors.Errorf("invalid rbd path: missing conf= in %s", diskPath)
	}
	keyringPath = filepath.Join(filepath.Dir(confPath), "ceph.keyring")
	return pool, image, confPath, keyringPath, nil
}

func imageSpec(pool, image string) string {
	return fmt.Sprintf("%s/%s", pool, image)
}

type rbdDeviceInfo struct {
	Id        int    `json:"id"`
	Pool      string `json:"pool"`
	Namespace string `json:"namespace"`
	Image     string `json:"image"`
	Name      string `json:"name"`
	Snap      string `json:"snap"`
	Device    string `json:"device"`
}

func (r rbd) listMappedDevices(confPath, keyringPath string) (map[string]string, error) {
	args := []string{"device", "list", "--format", "json"}
	if confPath != "" {
		args = append(args, "--conf", confPath)
	}
	if keyringPath != "" && fileutils2.Exists(keyringPath) {
		args = append(args, "--keyring", keyringPath)
	}
	out, err := procutils.NewRemoteCommandAsFarAsPossible("rbd", args...).Output()
	if err != nil {
		return nil, errors.Wrapf(err, "rbd device list: %s", string(out))
	}
	jsonObj, err := jsonutils.Parse(out)
	if err != nil {
		return nil, errors.Wrapf(err, "parse rbd device list json output: %s", string(out))
	}
	devices, err := jsonObj.GetArray()
	if err != nil {
		return nil, errors.Wrapf(err, "get devices array from json: %s", string(out))
	}
	result := make(map[string]string)
	for _, devObj := range devices {
		devInfo := rbdDeviceInfo{}
		if err := devObj.Unmarshal(&devInfo); err != nil {
			log.Warningf("failed to unmarshal device info: %v, skip", err)
			continue
		}
		image := devInfo.Image
		if image == "" {
			image = devInfo.Name
		}
		if devInfo.Pool == "" || image == "" || devInfo.Device == "" {
			continue
		}
		spec := imageSpec(devInfo.Pool, image)
		result[spec] = devInfo.Device
	}
	return result, nil
}

func (r rbd) CheckConnect(diskPath string) (string, bool, error) {
	pool, image, confPath, keyringPath, err := parseRbdPath(diskPath)
	if err != nil {
		return "", false, err
	}
	spec := imageSpec(pool, image)
	mapped, err := r.listMappedDevices(confPath, keyringPath)
	if err != nil {
		return "", false, err
	}
	dev, ok := mapped[spec]
	if !ok {
		return "", false, nil
	}
	devPath := r.checkPartition(dev)
	return devPath, true, nil
}

func (r rbd) checkPartition(devName string) string {
	// /dev/rbd0 -> /dev/rbd0p1
	partPath := devName + "p1"
	if fileutils2.Exists(partPath) {
		return partPath
	}
	return devName
}

func (r rbd) ConnectDisk(diskPath string) (string, error) {
	pool, image, confPath, keyringPath, err := parseRbdPath(diskPath)
	if err != nil {
		return "", err
	}
	spec := imageSpec(pool, image)
	args := []string{"device", "map", spec}
	if confPath != "" {
		args = append(args, "--conf", confPath)
	}
	if keyringPath != "" && fileutils2.Exists(keyringPath) {
		args = append(args, "--keyring", keyringPath)
	}
	out, err := procutils.NewRemoteCommandAsFarAsPossible("rbd", args...).Output()
	if err != nil {
		return "", errors.Wrapf(err, "rbd device map %s: %s", spec, string(out))
	}
	devStr := strings.TrimSpace(string(out))
	if devStr == "" {
		devPath, _, err := r.CheckConnect(diskPath)
		if err != nil || devPath == "" {
			return "", errors.Wrapf(err, "rbd map succeeded but device not found for %s", spec)
		}
		return r.checkPartition(devPath), nil
	}
	if !strings.HasPrefix(devStr, "/dev/") {
		devStr = "/dev/" + devStr
	}
	devStr = strings.TrimSpace(devStr)
	if idx := strings.Index(devStr, "\n"); idx > 0 {
		devStr = devStr[:idx]
	}
	return r.checkPartition(devStr), nil
}

func (r rbd) DisconnectDisk(diskPath string, mountPoint string) error {
	pool, image, confPath, keyringPath, err := parseRbdPath(diskPath)
	if err != nil {
		return err
	}
	spec := imageSpec(pool, image)
	mapped, err := r.listMappedDevices(confPath, keyringPath)
	if err != nil {
		log.Warningf("rbd device list before unmap: %v", err)
		return r.unmapBySpec(spec, confPath, keyringPath)
	}
	dev, ok := mapped[spec]
	if !ok {
		log.Infof("rbd image %s not mapped, skip unmap", spec)
		return nil
	}
	args := []string{"device", "unmap", dev}
	if confPath != "" {
		args = append(args, "--conf", confPath)
	}
	out, err := procutils.NewRemoteCommandAsFarAsPossible("rbd", args...).Output()
	if err != nil {
		if strings.Contains(string(out), "not mapped") || strings.Contains(string(out), "No such device") {
			return nil
		}
		return r.unmapBySpec(spec, confPath, keyringPath)
	}
	log.Infof("rbd device unmap %s (image %s) ok", dev, spec)
	return nil
}

func (r rbd) unmapBySpec(spec, confPath, keyringPath string) error {
	args := []string{"device", "unmap", spec}
	if confPath != "" {
		args = append(args, "--conf", confPath)
	}
	if keyringPath != "" && fileutils2.Exists(keyringPath) {
		args = append(args, "--keyring", keyringPath)
	}
	out, err := procutils.NewRemoteCommandAsFarAsPossible("rbd", args...).Output()
	if err != nil {
		return errors.Wrapf(err, "rbd device unmap %s: %s", spec, string(out))
	}
	return nil
}