// Copyright 2019 Yunion // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package hostinfo import ( "bufio" "context" "fmt" "os" "strconv" "strings" "time" "github.com/shirou/gopsutil/cpu" "github.com/shirou/gopsutil/mem" "yunion.io/x/jsonutils" "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/pkg/util/netutils" "yunion.io/x/pkg/util/regutils" computeapi "yunion.io/x/onecloud/pkg/apis/compute" hostapi "yunion.io/x/onecloud/pkg/apis/host" "yunion.io/x/onecloud/pkg/cloudcommon/types" "yunion.io/x/onecloud/pkg/hostman/hostinfo/hostbridge" "yunion.io/x/onecloud/pkg/hostman/hostinfo/hostdhcp" "yunion.io/x/onecloud/pkg/hostman/hostutils" "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/httperrors" modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute" "yunion.io/x/onecloud/pkg/util/fileutils2" "yunion.io/x/onecloud/pkg/util/netutils2" "yunion.io/x/onecloud/pkg/util/procutils" "yunion.io/x/onecloud/pkg/util/sysutils" ) type SCPUInfo struct { CpuCount int cpuFreq int64 // MHZ cpuFeatures []string CpuArchitecture string cpuInfoProc *types.SCPUInfo cpuInfoDmi *types.SDMICPUInfo } func DetectCpuInfo() (*SCPUInfo, error) { cpuinfo := new(SCPUInfo) cpuCount, _ := cpu.Counts(true) cpuinfo.CpuCount = cpuCount spec, err := cpuinfo.fetchCpuSpecs() if err != nil { return nil, err } var freq float64 strCpuFreq, ok := spec["cpu_freq"] if ok { freq, err = strconv.ParseFloat(strCpuFreq, 64) if err != nil { log.Errorln(err) return nil, err } } cpuinfo.cpuFreq = int64(freq) log.Infof("cpuinfo freq %d", cpuinfo.cpuFreq) cpuinfo.cpuFeatures = strings.Split(spec["flags"], " ") // cpu.Percent(interval, false) ret, err := fileutils2.FileGetContents("/proc/cpuinfo") if err != nil { return nil, errors.Wrap(err, "get cpuinfo") } cpuinfo.cpuInfoProc, err = sysutils.ParseCPUInfo(strings.Split(ret, "\n")) if err != nil { return nil, errors.Wrap(err, "parse cpu info") } bret, err := procutils.NewCommand("dmidecode", "-t", "4").Output() if err != nil { log.Errorf("dmidecode -t 4 error: %s(%s)", err, string(bret)) cpuinfo.cpuInfoDmi = &types.SDMICPUInfo{Nodes: 1} } else { cpuinfo.cpuInfoDmi = sysutils.ParseDMICPUInfo(strings.Split(string(bret), "\n")) } cpuArch, err := procutils.NewCommand("uname", "-m").Output() if err != nil { return nil, errors.Wrap(err, "get cpu architecture") } cpuinfo.CpuArchitecture = strings.TrimSpace(string(cpuArch)) return cpuinfo, nil } func (c *SCPUInfo) fetchCpuSpecs() (map[string]string, error) { f, err := os.Open("/proc/cpuinfo") if err != nil { return nil, err } defer f.Close() var spec = make(map[string]string, 0) scanner := bufio.NewScanner(f) for scanner.Scan() { line := scanner.Text() colon := strings.Index(line, ":") if colon > 0 { key := strings.TrimSpace(line[:colon]) val := strings.TrimSpace(line[colon+1:]) if key == "cpu MHz" { spec["cpu_freq"] = val } else if key == "flags" { spec["flags"] = val } } } if err := scanner.Err(); err != nil { log.Errorln(err) return nil, err } return spec, nil } type SMemory struct { Total int Free int Used int MemInfo *types.SDMIMemInfo } func DetectMemoryInfo() (*SMemory, error) { var smem = new(SMemory) info, err := mem.VirtualMemory() if err != nil { return nil, err } smem.Total = int(info.Total / 1024 / 1024) smem.Free = int(info.Available / 1024 / 1024) smem.Used = smem.Total - smem.Free ret, err := procutils.NewCommand("dmidecode", "-t", "17").Output() if err != nil { // ignore log.Errorf("dmidecode fail %s: %s", err, ret) smem.MemInfo = &types.SDMIMemInfo{} } else { smem.MemInfo = sysutils.ParseDMIMemInfo(strings.Split(string(ret), "\n")) } if smem.MemInfo.Total == 0 { // in case dmidecode is not work, use gopsutil smem.MemInfo.Total = smem.Total } return smem, nil } func (m *SMemory) GetHugepages() (sysutils.THugepages, error) { return sysutils.GetHugepages() } type SNIC struct { Inter string Bridge string Ip string Ip6 string Wire string WireId string Mask int Mask6 int Bandwidth int BridgeDev hostbridge.IBridgeDriver dhcpServer *hostdhcp.SGuestDHCPServer dhcpServer6 *hostdhcp.SGuestDHCP6Server } func (n *SNIC) EnableDHCPRelay() bool { if len(n.Ip) == 0 { return false } v4Ip, err := netutils.NewIPV4Addr(n.Ip) if err != nil { log.Errorf("EnableDHCPRelay netutils.NewIPV4Addr(%s) error: %v", n.Ip, err) return false } if len(options.HostOptions.DhcpRelay) == 2 && !netutils.IsExitAddress(v4Ip) { return true } else { return false } } func (n *SNIC) EnableDHCP6Relay() bool { if len(n.Ip6) == 0 { return false } _, err := netutils.NewIPV6Addr(n.Ip6) if err != nil { log.Errorf("EnableDHCP6Relay netutils.NewIPV6Addr(%s) error: %v", n.Ip6, err) return false } if len(options.HostOptions.Dhcp6Relay) == 2 { return true } else { return false } } func (n *SNIC) SetupDhcpRelay() error { if n.EnableDHCPRelay() { log.Infof("Enable dhcp relay on nic: %#v", n) if err := n.dhcpServer.RelaySetup(n.Ip); err != nil { return errors.Wrapf(err, "setup dhcp relay on ip: %s", n.Ip) } } if n.EnableDHCP6Relay() { log.Infof("Enable dhcpv6 relay on nic: %#v", n) if err := n.dhcpServer6.RelaySetup(n.Ip6); err != nil { return errors.Wrapf(err, "setup dhcpv6 relay on ip: %s", n.Ip6) } } return nil } func (n *SNIC) SetWireId(wire, wireId string, bandwidth int64) error { if len(n.Wire) == 0 { n.Wire = wire } else if n.Wire != wire { return errors.Wrapf(httperrors.ErrConflict, "expect wire %s != assign wire %s", n.Wire, wire) } else { // match } n.WireId = wireId n.Bandwidth = int(bandwidth) return nil } func (n *SNIC) ExitCleanup() { n.BridgeDev.CleanupConfig() log.Infof("Stop DHCP Server") // TODO stop dhcp server } func NewNIC(desc string) (*SNIC, error) { nic := new(SNIC) data := strings.Split(desc, "/") if len(data) < 3 { return nil, fmt.Errorf("Parse nic conf %s failed, too short", desc) } nic.Inter = data[0] nic.Bridge = data[1] if regutils.MatchIP4Addr(data[2]) { nic.Ip = data[2] if len(data) > 3 && regutils.MatchIP6Addr(data[3]) { nic.Ip6 = data[3] } } else if regutils.MatchIP6Addr(data[2]) { nic.Ip6 = data[2] if len(data) > 3 && regutils.MatchIP4Addr(data[3]) { nic.Ip = data[3] } } else { nic.Wire = data[2] } nic.Bandwidth = 1000 log.Infof("IP %s/%s/%s/%s", nic.Ip, nic.Ip6, nic.Bridge, nic.Inter) // fetch ip and ip6 netmask from interface and bridge if len(nic.Ip) > 0 || len(nic.Ip6) > 0 { // waiting for interface assign ip // in case nic bonding is too slow var max, wait = 30, 0 for wait < max { inf := netutils2.NewNetInterfaceWithExpectIp(nic.Inter, nic.Ip, nic.Ip6, nil) if len(nic.Ip) > 0 && inf.Addr == nic.Ip { mask, _ := inf.Mask.Size() if mask > 0 { nic.Mask = mask } } if len(nic.Ip6) > 0 && inf.Addr6 == nic.Ip6 { mask, _ := inf.Mask6.Size() if mask > 0 { nic.Mask6 = mask } } br := netutils2.NewNetInterface(nic.Bridge) if br.Addr == nic.Ip { mask, _ := br.Mask.Size() if nic.Mask == 0 && mask > 0 { nic.Mask = mask } } if br.Addr6 == nic.Ip6 { mask, _ := br.Mask6.Size() if nic.Mask6 == 0 && mask > 0 { nic.Mask6 = mask } } if nic.Mask > 0 || nic.Mask6 > 0 { break } time.Sleep(time.Second * 2) wait += 1 } if wait >= max { // if ip not found in inter or bridge return nil, fmt.Errorf("Ip %s is not configure on %s/%s ?", nic.Ip, nic.Bridge, nic.Inter) } } var err error nic.BridgeDev, err = hostbridge.NewDriver(options.HostOptions.BridgeDriver, nic.Bridge, nic.Inter, nic.Ip, nic.Mask, nic.Ip6, nic.Mask6) if err != nil { return nil, errors.Wrapf(err, "hostbridge.NewDriver driver: %s, bridge: %s, interface: %s, ip: %s/%d, ip6: %s/%d", options.HostOptions.BridgeDriver, nic.Bridge, nic.Inter, nic.Ip, nic.Mask, nic.Ip6, nic.Mask6) } confirm, msg, err := nic.BridgeDev.ConfirmToConfig() if err != nil { return nil, errors.Wrapf(err, "nic.BridgeDev.ConfirmToConfig %#v", nic.BridgeDev) } if !confirm { log.Infof("Not confirm to configuration, bridge %s reason: %s", nic.Bridge, msg) if err = nic.BridgeDev.Setup(nic.BridgeDev); err != nil { return nil, errors.Wrapf(err, "nic.BridgeDev.Setup %v", nic.BridgeDev) } time.Sleep(time.Second * 1) } else { log.Infof("Confirm to configuration!! To migrate physical interface configs") err := nic.BridgeDev.MigrateSlaveConfigs(nic.BridgeDev) if err != nil { log.Errorf("fail to migrate configs: %s", err) } } if err := nic.BridgeDev.PersistentConfig(); err != nil { return nil, errors.Wrapf(err, "nic.BridgeDev.PersistentConfig %v", nic.BridgeDev) } if isDHCP, err := nic.BridgeDev.DisableDHCPClient(); err != nil { return nil, errors.Wrap(err, "disable dhcp client") } else if isDHCP { Instance().AppendHostError("dhcp client is enabled before host agent start, please disable it") } var relayConf *hostdhcp.SDHCPRelayUpstream if nic.EnableDHCPRelay() { log.Infof("EnableDHCPRelay on nic %#v", nic) relayConf = &hostdhcp.SDHCPRelayUpstream{} relayConf.IP = options.HostOptions.DhcpRelay[0] relayConf.Port, err = strconv.Atoi(options.HostOptions.DhcpRelay[1]) if err != nil { return nil, errors.Wrapf(err, "invalid relay port %s", options.HostOptions.DhcpRelay[1]) } } nic.dhcpServer, err = hostdhcp.NewGuestDHCPServer(nic.Bridge, options.HostOptions.DhcpServerPort, relayConf) if err != nil { return nil, errors.Wrapf(err, "NewGuestDHCPServer(%s, %d, %#v)", nic.Bridge, options.HostOptions.DhcpServerPort, relayConf) } var relayConf6 *hostdhcp.SDHCPRelayUpstream if nic.EnableDHCP6Relay() { log.Infof("EnableDHCP6Relay on nic %#v", nic) relayConf6 = &hostdhcp.SDHCPRelayUpstream{} relayConf6.IP = options.HostOptions.Dhcp6Relay[0] relayConf6.Port, err = strconv.Atoi(options.HostOptions.Dhcp6Relay[1]) if err != nil { return nil, errors.Wrapf(err, "invalid relay port %s", options.HostOptions.Dhcp6Relay[1]) } } if !nic.BridgeDev.IsV4Only() { nic.dhcpServer6, err = hostdhcp.NewGuestDHCP6Server(nic.Bridge, options.HostOptions.Dhcp6ServerPort, relayConf6) if err != nil { return nil, errors.Wrapf(err, "NewGuestDHCP6Server(%s, %d, %#v)", nic.Bridge, options.HostOptions.Dhcp6ServerPort, relayConf6) } } // dhcp server start after guest manager init return nic, nil } func (n *SNIC) IsHostLocal() bool { if n.Bridge == options.HostOptions.HostLocalBridgeName { n.setupHostLocal() return true } return false } func (n *SNIC) setupHostLocal() { if n.WireId == "" { n.Wire = computeapi.DEFAULT_HOST_LOCAL_WIRE_NAME n.WireId = computeapi.DEFAULT_HOST_LOCAL_WIRE_ID } } type SSysInfo struct { *types.SSystemInfo Nest string `json:"nest,omitempty"` OsDistribution string `json:"os_distribution"` OsVersion string `json:"os_version"` KernelVersion string `json:"kernel_version"` QemuVersion string `json:"qemu_version"` OvsVersion string `json:"ovs_version"` OvsKmodVersion string `json:"ovs_kmod_version"` KvmModule string `json:"kvm_module"` CpuModelName string `json:"cpu_model_name"` CpuMicrocode string `json:"cpu_microcode"` CgroupVersion string `json:"cgroup_version"` StorageType string `json:"storage_type"` HugepagesOption string `json:"hugepages_option"` HugepageSizeKb int `json:"hugepage_size_kb"` HugepageNr *int `json:"hugepage_nr"` NodeHugepages []hostapi.HostNodeHugepageNr `json:"node_hugepages"` EnableKsm bool `json:"enable_ksm"` HostAgentCpuNumaAllocate bool `json:"host_agent_cpu_numa_allocate"` Topology *hostapi.HostTopology `json:"topology"` CPUInfo *hostapi.HostCPUInfo `json:"cpu_info"` MotherboardInfo *types.SSystemInfo `json:"motherboard_info"` } func StartDetachStorages(hs []jsonutils.JSONObject) { for len(hs) > 0 { hostId, _ := hs[0].GetString("host_id") storageId, _ := hs[0].GetString("storage_id") _, err := modules.Hoststorages.Detach( hostutils.GetComputeSession(context.Background()), hostId, storageId, nil) if err != nil { log.Errorf("Host %s detach storage %s failed: %s", hostId, storageId, err) time.Sleep(30 * time.Second) } else { hs = hs[1:] } } } func IsRootPartition(path string) bool { if !strings.HasPrefix(path, "/") { return false } path = strings.TrimSuffix(path, "/") pathSegs := strings.Split(path, "/") for len(pathSegs) > 1 { err := procutils.NewRemoteCommandAsFarAsPossible("mountpoint", path).Run() if err != nil { pathSegs = pathSegs[:len(pathSegs)-1] path = strings.Join(pathSegs, "/") continue } else { return false } } return true }