// Copyright 2019 Yunion // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // // http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. package hostbridge import ( "fmt" "strings" "time" "yunion.io/x/log" "yunion.io/x/pkg/errors" "yunion.io/x/pkg/utils" "yunion.io/x/onecloud/pkg/apis/compute" "yunion.io/x/onecloud/pkg/hostman/guestman/desc" "yunion.io/x/onecloud/pkg/hostman/options" "yunion.io/x/onecloud/pkg/hostman/system_service" "yunion.io/x/onecloud/pkg/util/bwutils" "yunion.io/x/onecloud/pkg/util/procutils" ) type SOVSBridgeDriver struct { SBaseBridgeDriver } func (o *SOVSBridgeDriver) CleanupConfig() { //ovsutils.CleanAllHiddenPorts() // if enableopenflowcontroller ... } func (o *SOVSBridgeDriver) Exists() (bool, error) { data, err := procutils.NewCommand("ovs-vsctl", "list-br").Output() if err != nil { return false, errors.Wrapf(err, "failed list br: %s", data) } for _, d := range strings.Split(string(data), "\n") { if strings.TrimSpace(d) == o.bridge.String() { return true, nil } } return false, nil } func (o *SOVSBridgeDriver) Interfaces() ([]string, error) { data, err := procutils.NewCommand("ovs-vsctl", "list-ifaces", o.bridge.String()).Output() if err != nil { return nil, errors.Wrapf(err, "failed list ifaces: %s", data) } var infs = make([]string, 0) for _, d := range strings.Split(string(data), "\n") { if len(strings.TrimSpace(d)) > 0 { infs = append(infs, strings.TrimSpace(d)) } } return infs, nil } func (o *SOVSBridgeDriver) SetupInterface() error { infs, err := o.Interfaces() if err != nil { return err } if o.inter != nil && !utils.IsInStringArray(o.inter.String(), infs) { output, err := procutils.NewCommand("ovs-vsctl", "--", "--may-exist", "add-port", o.bridge.String(), o.inter.String()).Output() if err != nil { return fmt.Errorf("Failed to add interface: %s, %s", err, output) } } return nil } func (o *SOVSBridgeDriver) SetupBridgeDev() error { exist, err := o.Exists() if err != nil { return err } if !exist { output, err := procutils.NewCommand("ovs-vsctl", "--", "--may-exist", "add-br", o.bridge.String()).Output() if err != nil { return errors.Wrapf(err, "ovs-vsctl add br %s failed: %s", o.bridge.String(), output) } return nil } return nil } func (d *SOVSBridgeDriver) PersistentConfig() error { if d.inter == nil { return nil } args := []string{ "ovs-vsctl", "set", "Bridge", d.bridge.String(), "other-config:hwaddr=" + d.inter.GetMac(), } output, err := procutils.NewCommand(args[0], args[1:]...).Output() if err != nil { return fmt.Errorf("Ovs bridge set mac address failed %s %s", output, err) } args = []string{ "ovs-vsctl", "set", "Interface", d.bridge.String(), fmt.Sprintf("mtu_request=%d", d.inter.Mtu), } output, err = procutils.NewCommand(args[0], args[1:]...).Output() if err != nil { return fmt.Errorf("Ovs bridge set MTU failed %s %s", output, err) } return nil } func (o *SOVSBridgeDriver) GenerateIfdownScripts(scriptPath string, nic *desc.SGuestNetwork, isVolatileHost bool) error { return o.generateIfdownScripts(o, scriptPath, nic, isVolatileHost) } func (o *SOVSBridgeDriver) GenerateIfupScripts(scriptPath string, nic *desc.SGuestNetwork, isVolatileHost bool) error { return o.generateIfupScripts(o, scriptPath, nic, isVolatileHost) } func (o *SOVSBridgeDriver) OnVolatileGuestResume(nic *desc.SGuestNetwork) error { cmd := []string{"ovs-vsctl", "set", "Interface", nic.Ifname, fmt.Sprintf("external_ids:iface-id=iface-%s-%s", nic.NetId, nic.Ifname), } output, err := procutils.NewRemoteCommandAsFarAsPossible(cmd[0], cmd[1:]...).Output() if err != nil { log.Errorf("failed exec %v: %s %s", cmd, err, output) return errors.Wrapf(err, "set interface external_ids failed: %s", output) } return nil } func (o *SOVSBridgeDriver) getUpScripts(nic *desc.SGuestNetwork, isVolatileHost bool) (string, error) { var ( bridge = o.bridge.String() ifname = nic.Ifname ip = nic.Ip mac = nic.Mac netId = nic.NetId vlan = nic.Vlan vpcProvider = nic.Vpc.Provider ) if vpcProvider == compute.VPC_PROVIDER_OVN { bridge = options.HostOptions.OvnIntegrationBridge } s := "#!/bin/bash\n\n" s += fmt.Sprintf("SWITCH='%s'\n", bridge) s += fmt.Sprintf("IF='%s'\n", ifname) s += fmt.Sprintf("IP='%s'\n", ip) s += fmt.Sprintf("MAC='%s'\n", mac) s += fmt.Sprintf("VLAN_ID=%d\n", vlan) s += fmt.Sprintf("NET_ID=%s\n", netId) limit, burst, err := bwutils.GetOvsBwValues(nic.Bw, nic.Ip) if err != nil { return "", err } s += fmt.Sprintf("LIMIT=%d\n", limit) s += fmt.Sprintf("BURST=%d\n", burst) bwDownload, err := bwutils.GetDownloadBwValue(nic.Bw, nic.Ip, nic.Ifname, options.HostOptions.BwDownloadBandwidth) if err != nil { return "", err } s += fmt.Sprintf("LIMIT_DOWNLOAD='%dmbit'\n", bwDownload) if options.HostOptions.TunnelPaddingBytes > 0 { s += fmt.Sprintf("ip link set dev $IF mtu %d\n", 1500+options.HostOptions.TunnelPaddingBytes) } s += "ip address flush dev $IF\n" s += "ip link set dev $IF up\n" s += "ovs-vsctl list-ifaces $SWITCH | grep -w $IF > /dev/null 2>&1\n" s += "if [ $? -eq '0' ]; then\n" s += " ovs-vsctl del-port $SWITCH $IF\n" s += "fi\n" s += "if [ \"$VLAN_ID\" -ne \"1\" ]; then\n" s += " TAG=\"tag=$VLAN_ID\"\n" s += "fi\n" s += "ovs-vsctl add-port $SWITCH $IF $TAG\n" if vpcProvider == compute.VPC_PROVIDER_OVN && !isVolatileHost { s += "ovs-vsctl set Interface $IF external_ids:iface-id=iface-$NET_ID-$IF\n" } s += "PORT=$(ovs-ofctl show $SWITCH | grep -w $IF)\n" s += "PORT=$(echo $PORT | awk 'BEGIN{FS=\"(\"}{print $1}')\n" s += "OFCTL=$(ovs-vsctl get-controller $SWITCH)\n" if nic.Driver != compute.NETWORK_DRIVER_VFIO { s += "if [ -z \"$OFCTL\" ]; then\n" s += " ovs-vsctl set Interface $IF ingress_policing_rate=$LIMIT\n" s += " ovs-vsctl set Interface $IF ingress_policing_burst=$BURST\n" s += "fi\n" if vpcProvider != compute.VPC_PROVIDER_OVN && len(options.HostOptions.SRIOVNics) > 0 { s += fmt.Sprintf("bridge fdb add %s dev %s\n", nic.Mac, o.inter.String()) } } s += "if [ $LIMIT_DOWNLOAD != \"0mbit\" ]; then\n" s += " tc qdisc del dev $IF root 2>/dev/null\n" s += " tc qdisc add dev $IF root handle 1: htb default 10\n" s += " tc class add dev $IF parent 1: classid 1:1 htb " + "rate $LIMIT_DOWNLOAD ceil $LIMIT_DOWNLOAD\n" s += " tc class add dev $IF parent 1:1 classid 1:10 htb " + "rate $LIMIT_DOWNLOAD ceil $LIMIT_DOWNLOAD\n" s += "fi\n" return s, nil } func (o *SOVSBridgeDriver) getDownScripts(nic *desc.SGuestNetwork, isVolatileHost bool) (string, error) { var ( bridge = o.bridge.String() ifname = nic.Ifname ip = nic.Ip mac = nic.Mac vlan = nic.Vlan ) s := "#!/bin/bash\n\n" s += fmt.Sprintf("SWITCH='%s'\n", bridge) s += fmt.Sprintf("IF='%s'\n", ifname) s += fmt.Sprintf("IP='%s'\n", ip) s += fmt.Sprintf("MAC='%s'\n", mac) s += fmt.Sprintf("VLAN_ID=%d\n", vlan) s += "PORT=$(ovs-ofctl show $SWITCH | grep -w $IF)\n" s += "if [ $? -ne '0' ]; then\n" s += " exit 0\n" s += "fi\n" s += "OFCTL=$(ovs-vsctl get-controller $SWITCH)\n" s += "PORT=$(echo $PORT | awk 'BEGIN{FS=\"(\"}{print $1}')\n" s += "ip link set dev $IF down\n" s += "ovs-vsctl -- --if-exists del-port $SWITCH $IF\n" if nic.Driver != compute.NETWORK_DRIVER_VFIO { if nic.Vpc.Provider != compute.VPC_PROVIDER_OVN && len(options.HostOptions.SRIOVNics) > 0 { s += fmt.Sprintf("bridge fdb del %s dev %s\n", nic.Mac, o.inter.String()) } } return s, nil } type SRule struct { priority int cond string actions string } func (o *SOVSBridgeDriver) RegisterHostlocalServer(mac, ip string) error { return nil } func (o *SOVSBridgeDriver) ovsSetParams(params map[string]map[string]string) { for tbl, tblval := range params { for k, v := range tblval { procutils.NewCommand("ovs-vsctl", "set", tbl, o.bridge.String(), fmt.Sprintf("%s=%s", k, v)).Run() } } } func (o *SOVSBridgeDriver) WarmupConfig() error { // if options.OvsSflowBridges ... if options.HostOptions.EnableOpenflowController { // ... } else { params := map[string]map[string]string{ "bridge": { "stp_enable": "false", "fail_mode": "standalone", "other-config:flow-eviction-threshold": "2500", }, } o.ovsSetParams(params) } return nil } func OVSPrepare() error { ovs := system_service.GetService("openvswitch") if !ovs.IsInstalled() || !ovs.IsActive() { // no openvswitch service found, first try load openvswitch kernel modules, then try ovs-vsctl command, if success, return nil if !utils.IsInStringArray("openvswitch", options.HostOptions.SkipCheckKernelMods) { err := procutils.NewRemoteCommandAsFarAsPossible("modprobe", "openvswitch").Run() if err != nil { return errors.Wrap(err, "Failed to load openvswitch kernel modules") } } // wait for the ovs-vswitchd to start startProbe := time.Now() const waitSeconds = 60 * 2 // wait ovs-vswitch running for 2 minutes for time.Since(startProbe) < time.Duration(waitSeconds)*time.Second { err := procutils.NewCommand("ovs-vsctl", "show").Run() if err != nil { time.Sleep(2 * time.Second) } else { return nil } } // ovs service start timeout if !ovs.IsInstalled() { // ovs service not installed, return error return fmt.Errorf("service openvswitch not installed and wait ovs service timeout, please check ovs service status") } // ovs service installed but not running, continue to start the service } if ovs.IsEnabled() { err := ovs.Disable() if err != nil { log.Errorf("Disabling openvswitch service failed: %v", err) } } if !ovs.IsActive() { return ovs.Start(false) } return nil } func cleanOvsBridge() { //ovsutils.CleanAllHiddenPorts() } func NewOVSBridgeDriver(bridge, inter, ip string, maskLen int, ip6 string, mask6Len int) (*SOVSBridgeDriver, error) { base, err := NewBaseBridgeDriver(bridge, inter, ip, maskLen, ip6, mask6Len) if err != nil { return nil, err } ovsDrv := &SOVSBridgeDriver{*base} ovsDrv.drv = ovsDrv return ovsDrv, nil } func NewOVSBridgeDriverByName(bridge string) (*SOVSBridgeDriver, error) { return NewOVSBridgeDriver(bridge, "", "", 0, "", 0) }