// Copyright 2019 Yunion // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package container_device import ( "fmt" "regexp" "strconv" "strings" runtimeapi "k8s.io/cri-api/pkg/apis/runtime/v1" "yunion.io/x/jsonutils" "yunion.io/x/pkg/errors" hostapi "yunion.io/x/onecloud/pkg/apis/host" "yunion.io/x/onecloud/pkg/hostman/isolated_device" "yunion.io/x/onecloud/pkg/httperrors" "yunion.io/x/onecloud/pkg/util/fileutils2" "yunion.io/x/onecloud/pkg/util/procutils" ) var ( NetintCAASICReg = regexp.MustCompile("T4\\d\\d-.*") NetintCAQuadraReg = regexp.MustCompile("Quadra.*") ) const ( NETINT_VENDOR_ID = "0000" NETINT_DEVICE_ID = "0000" ) func init() { isolated_device.RegisterContainerDeviceManager(newNetintDeviceManager(isolated_device.ContainerNetintCAASIC, NetintCAASICReg)) isolated_device.RegisterContainerDeviceManager(newNetintDeviceManager(isolated_device.ContainerNetintCAQuadra, NetintCAQuadraReg)) } type NetintDeviceInfo struct { Namespace int `json:"NameSpace"` DevicePath string `json:"DevicePath"` Firmware string `json:"Firmware"` Index int `json:"Index"` ModelNumber string `json:"ModelNumber"` ProductName string `json:"ProductName"` SerialNumber string `json:"SerialNumber"` UsedBytes int `json:"UsedBytes"` MaximumLBA int `json:"MaximumLBA"` PhysicalSize int `json:"PhysicalSize"` SectorSize int `json:"SectorSize"` } func (info NetintDeviceInfo) GetIndexFromPath() (int, error) { idxStr := strings.TrimSuffix(strings.TrimPrefix(info.DevicePath, "/dev/nvme"), "n1") idx, err := strconv.Atoi(idxStr) if err != nil { return -1, errors.Wrapf(err, "strconv.Atoi(%s) by %q", idxStr, info.DevicePath) } return idx, nil } func (info NetintDeviceInfo) GetIndex() (int, error) { if info.Index > 0 { return info.Index, nil } idx, err := info.GetIndexFromPath() if err != nil { return -1, err } return idx, nil } type netintDeviceManager struct { devType isolated_device.ContainerDeviceType devRegPattern *regexp.Regexp } func newNetintDeviceManager(devType isolated_device.ContainerDeviceType, reg *regexp.Regexp) *netintDeviceManager { return &netintDeviceManager{ devType: devType, devRegPattern: reg, } } func (m *netintDeviceManager) GetType() isolated_device.ContainerDeviceType { return m.devType } func (m *netintDeviceManager) ProbeDevices() ([]isolated_device.IDevice, error) { return nil, nil } type NVMEListResult struct { Devices []*NetintDeviceInfo `json:"devices"` } func (m *netintDeviceManager) fetchNVMEDevices() ([]*NetintDeviceInfo, error) { out, err := procutils.NewRemoteCommandAsFarAsPossible("bash", "-c", "nvme list -o json").Output() if err != nil { return nil, errors.Wrap(err, "get nvme device json output") } obj, err := jsonutils.Parse(out) if err != nil { return nil, errors.Wrapf(err, "jsonutils.Parse %s", string(out)) } output := new(NVMEListResult) if err := obj.Unmarshal(&output); err != nil { return nil, errors.Wrapf(err, "Unmarshal to NetIntDeviceInfo: %s", obj.String()) } result := make([]*NetintDeviceInfo, 0) for _, dev := range output.Devices { if !m.devRegPattern.MatchString(dev.ModelNumber) { continue } tmpDev := dev result = append(result, tmpDev) } return result, nil } func (m *netintDeviceManager) checkDevPath(devs []*NetintDeviceInfo, dev *isolated_device.ContainerDevice) error { if dev.Path == "" { return nil } isFound := false for _, d := range devs { if d.DevicePath == dev.Path { isFound = true break } } if isFound { return nil } return fmt.Errorf("%s not found in %s", dev.Path, jsonutils.Marshal(devs)) } func (m *netintDeviceManager) NewDevices(dev *isolated_device.ContainerDevice) ([]isolated_device.IDevice, error) { if err := CheckVirtualNumber(dev); err != nil { return nil, err } nvmeDevs, err := m.fetchNVMEDevices() if err != nil { return nil, errors.Wrap(err, "fetch nvme devices") } if err := m.checkDevPath(nvmeDevs, dev); err != nil { return nil, errors.Wrap(err, "check nvme devices") } result := make([]isolated_device.IDevice, 0) for _, nvmeDev := range nvmeDevs { for i := 0; i < dev.VirtualNumber; i++ { newDev, err := m.newDeviceByIndex(nvmeDev, i) if err != nil { return nil, errors.Wrapf(err, "newDeviceByIndex %#v %d", nvmeDev, i) } result = append(result, newDev) } } return result, nil } func (m *netintDeviceManager) newDeviceByIndex(dev *NetintDeviceInfo, idx int) (*netintDevice, error) { devIdx, err := dev.GetIndex() if err != nil { return nil, errors.Wrap(err, "dev.GetIndex") } devInfo := &isolated_device.PCIDevice{ Addr: fmt.Sprintf("%d-%d", devIdx, idx), VendorId: NETINT_VENDOR_ID, DeviceId: NETINT_DEVICE_ID, ModelName: dev.ModelNumber, } nvmeDev := &netintDevice{ BaseDevice: NewBaseDevice(devInfo, m.devType, dev.DevicePath), info: dev, } return nvmeDev, nil } func (m *netintDeviceManager) NewContainerDevices(_ *hostapi.ContainerCreateInput, input *hostapi.ContainerDevice) ([]*runtimeapi.Device, []*runtimeapi.Device, error) { dev := input.IsolatedDevice if !fileutils2.Exists(dev.Path) { return nil, nil, errors.Wrapf(httperrors.ErrNotFound, "device path %s doesn't exist", dev.Path) } charDevReg := regexp.MustCompile("(.*)n\\d+") charDevPath := charDevReg.FindAllStringSubmatch(dev.Path, -1)[0][1] ctrDevs := []*runtimeapi.Device{ { HostPath: dev.Path, ContainerPath: dev.Path, Permissions: "rwm", }, { HostPath: charDevPath, ContainerPath: charDevPath, Permissions: "rwm", }, } return ctrDevs, nil, nil } func (m *netintDeviceManager) GetContainerExtraConfigures(devs []*hostapi.ContainerDevice) ([]*runtimeapi.KeyValue, []*runtimeapi.Mount) { return nil, nil } type netintDevice struct { *BaseDevice info *NetintDeviceInfo } func (d netintDevice) GetNVMESizeMB() int { return d.info.PhysicalSize / 1024 / 1024 }