Cloudpods/backend/pkg/hostman/diskutils/nbd/nbdman.go

// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package nbd

import (
	"fmt"
	"sync"
	"time"

	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"

	"yunion.io/x/onecloud/pkg/util/fileutils2"
	"yunion.io/x/onecloud/pkg/util/procutils"
	"yunion.io/x/onecloud/pkg/util/sysutils"
)

type SNBDManager struct {
	nbdDevs map[string]bool
	nbdChan chan struct{}
	nbdLock *sync.Mutex
}

var nbdManager *SNBDManager

func Init() error {
	var err error
	nbdManager, err = NewNBDManager()
	return err
}

func GetNBDManager() *SNBDManager {
	return nbdManager
}

func NewNBDManager() (*SNBDManager, error) {
	var ret = new(SNBDManager)
	ret.nbdDevs = make(map[string]bool, 0)
	ret.nbdLock = new(sync.Mutex)

	ret.cleanupNbdDevices()
	if err := ret.reloadNbdDevices(); err != nil {
		return ret, errors.Wrap(err, "reloadNbdDevices")
	}
	if err := ret.findNbdDevices(); err != nil {
		return ret, errors.Wrap(err, "findNbdDevices")
	}
	ret.nbdChan = make(chan struct{}, len(ret.nbdDevs))

	return ret, nil
}

func tryDetachNbd(nbddev string) error {
	const MaxTries = 3
	tried := 0
	var errs []error
	for tried < MaxTries && fileutils2.IsBlockDeviceUsed(nbddev) {
		tried++
		if err := putdownNbdDevice(nbddev); err != nil {
			errs = append(errs, err)
			time.Sleep(time.Second)
			continue
		}
		break

	}
	if tried < MaxTries {
		return nil
	}
	if len(errs) > 0 {
		return errors.NewAggregate(errs)
	}
	return nil
}

func putdownNbdDevice(nbddev string) error {
	nbdDriver := &NBDDriver{nbdDev: nbddev}

	if err := nbdDriver.findPartitions(); err != nil {
		return err
	}

	if _, err := nbdDriver.setupLVMS(); err != nil {
		return err
	}
	for i := range nbdDriver.partitions {
		if nbdDriver.partitions[i].IsMounted() {
			if err := nbdDriver.partitions[i].Umount(); err != nil {
				return errors.Wrapf(err, "umount %s", nbdDriver.partitions[i].GetPartDev())
			}
		}
	}

	if !nbdDriver.putdownLVMs() {
		return errors.Errorf("failed putdown lvms")
	}
	if err := QemuNbdDisconnect(nbddev); err != nil {
		return err
	}
	return nil

}

func (m *SNBDManager) cleanupNbdDevices() {
	var i = 0
	for {
		nbddev := fmt.Sprintf("/dev/nbd%d", i)
		if fileutils2.Exists(nbddev) {
			if fileutils2.IsBlockDeviceUsed(nbddev) {
				log.Infof("nbd device %s is used", nbddev)
				err := tryDetachNbd(nbddev)
				if err != nil {
					log.Errorf("tryDetachNbd fail %s", err)
				} else {
					m.ReleaseNbddev(nbddev)
				}
			}
			i++
		} else {
			break
		}
	}
}

func (m *SNBDManager) reloadNbdDevices() error {
	output, err := procutils.NewRemoteCommandAsFarAsPossible("rmmod", "nbd").Output()
	if err != nil {
		log.Errorf("rmmod error: %s", output)
	}
	output, err = procutils.NewRemoteCommandAsFarAsPossible("modprobe", "nbd", "max_part=16").Output()
	if err != nil {
		return errors.Wrapf(err, "Failed to activate nbd device: %s", output)
	}
	return nil
}

func (m *SNBDManager) findNbdDevices() error {
	var i = 0
	for {
		nbddev := fmt.Sprintf("/dev/nbd%d", i)
		if fileutils2.Exists(nbddev) {
			i++
			if fileutils2.IsBlockDeviceUsed(nbddev) {
				continue
			}
			m.nbdDevs[nbddev] = false
			// https://www.kernel.org/doc/Documentation/ABI/testing/sysfs-class-bdi
			nbdBdi := fmt.Sprintf("/sys/block/nbd%d/bdi/", i-1)
			sysutils.SetSysConfig(nbdBdi+"max_ratio", "0")
			sysutils.SetSysConfig(nbdBdi+"min_ratio", "0")
		} else {
			break
		}
	}
	if len(m.nbdDevs) == 0 {
		return errors.Wrap(errors.ErrNotFound, "No nbd devices found")
	}
	log.Infof("NBD_DEVS: %#v", m.nbdDevs)
	return nil
}

func (m *SNBDManager) AcquireNbddev() string {
	if len(m.nbdDevs) == 0 {
		return ""
	}

	m.nbdChan <- struct{}{}
	defer m.nbdLock.Unlock()
	m.nbdLock.Lock()

	for nbdDev := range m.nbdDevs {
		if fileutils2.IsBlockDeviceUsed(nbdDev) {
			m.nbdDevs[nbdDev] = true
			continue
		}
		if !m.nbdDevs[nbdDev] {
			m.nbdDevs[nbdDev] = true
			return nbdDev
		}
	}
	return ""
}

func (m *SNBDManager) ReleaseNbddev(nbddev string) {
	if _, ok := m.nbdDevs[nbddev]; ok {
		defer m.nbdLock.Unlock()
		m.nbdLock.Lock()
		if !fileutils2.IsBlockDeviceUsed(nbddev) {
			m.nbdDevs[nbddev] = false
		}
	}
	<-m.nbdChan
}