Cloudpods/backend/pkg/hostman/hostinfo/finalize_network.go

// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package hostinfo

import (
	"context"
	"fmt"

	"yunion.io/x/jsonutils"
	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"
	"yunion.io/x/pkg/util/netutils"

	computeapis "yunion.io/x/onecloud/pkg/apis/compute"
	"yunion.io/x/onecloud/pkg/hostman/hostutils"
	"yunion.io/x/onecloud/pkg/hostman/options"
	computemodules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
	computeoptions "yunion.io/x/onecloud/pkg/mcclient/options/compute"
	"yunion.io/x/onecloud/pkg/util/fileutils2"
	"yunion.io/x/onecloud/pkg/util/iproute2"
	"yunion.io/x/onecloud/pkg/util/netutils2"
	"yunion.io/x/onecloud/pkg/util/procutils"
)

func (h *SHostInfo) findExternalInterfaces() []string {
	interfaces := make([]string, 0)
	for i := 0; i < len(h.Nics); i++ {
		if !h.Nics[i].IsHostLocal() && len(h.Nics[i].Ip) > 0 {
			interfaces = append(interfaces, h.Nics[i].Bridge)
		}
	}
	if len(options.HostOptions.ListenInterface) > 0 {
		interfaces = append(interfaces, options.HostOptions.ListenInterface)
	}
	return interfaces
}

func (h *SHostInfo) finalizeNetworkSetup(ctx context.Context) error {
	for i := 0; i < len(h.Nics); i++ {
		if err := h.Nics[i].SetupDhcpRelay(); err != nil {
			return errors.Wrapf(err, "SetupDhcpRelay %s", h.Nics[i])
		}
	}
	extInterfaces := h.findExternalInterfaces()
	for i := 0; i < len(h.Nics); i++ {
		if err := h.Nics[i].setupHostLocalNetworks(ctx, extInterfaces); err != nil {
			return errors.Wrapf(err, "SetupHostLocalNetworks %s", h.Nics[i])
		}
	}
	return nil
}

func (n *SNIC) String() string {
	return fmt.Sprintf("%s/%s/%s", n.Inter, n.Bridge, n.Ip)
}

func (n *SNIC) setupHostLocalNetworks(ctx context.Context, extInterfaces []string) error {
	nets, err := n.fetchHostLocalNetworks(ctx)
	if err != nil {
		return errors.Wrap(err, "fetchHostLocalNetworks")
	}
	hostLocalNics := make([]computeapis.NetworkDetails, 0)
	for i := range nets {
		net := nets[i]
		if len(net.GuestGateway) == 0 {
			continue
		}
		err := n.setupHostLocalNet(ctx, net, extInterfaces)
		if err != nil {
			return errors.Wrapf(err, "setupHostLocalNet of %s", jsonutils.Marshal(net))
		}
		hostLocalNics = append(hostLocalNics, nets[i])
	}
	// save hostLocalNics
	fn := options.HostOptions.HostLocalNetconfPath(n.Bridge)
	fileutils2.FilePutContents(fn, jsonutils.Marshal(hostLocalNics).PrettyString(), false)
	return nil
}

func (n *SNIC) setupHostLocalNet(ctx context.Context, netInfo computeapis.NetworkDetails, extInterfaces []string) error {
	// setup gateway ip
	if err := n.setupSlaveIp(ctx, netInfo.GuestGateway, netInfo.GuestIpMask, extInterfaces); err != nil {
		return errors.Wrapf(err, "setupSlaveIp %s %s", n, netInfo.GuestGateway)
	}
	return nil
}

func (n *SNIC) setupSlaveIp(ctx context.Context, gatewayIp string, maskLen byte, extInterfaces []string) error {
	bridgeIf := netutils2.NewNetInterface(n.Bridge)
	slaveAddrs := bridgeIf.GetSlaveAddresses()
	var curGatewayIp string
	var curMaskLen int
	isMaskUpdate := false
	for i := range slaveAddrs {
		addr := slaveAddrs[i]
		curGatewayIp = addr.Addr
		curMaskLen = addr.MaskLen
		if curGatewayIp == gatewayIp {
			if curMaskLen == int(maskLen) {
				// already configured, skip
				return nil
			} else {
				isMaskUpdate = true
				break
			}
		}
	}
	if err := n.BridgeDev.SetupSlaveAddresses([]netutils2.SNicAddress{
		{Addr: gatewayIp, MaskLen: int(maskLen)},
	}); err != nil {
		return errors.Wrap(err, "SetupSlaveAddresses")
	}
	if err := n.setupMasqueradeRule(ctx, gatewayIp, maskLen, extInterfaces); err != nil {
		return errors.Wrap(err, "setupMasqueradeRule")
	}
	if isMaskUpdate {
		brName := n.BridgeDev.Bridge()
		logPrefix := fmt.Sprintf("%s slave address %s mask is update from %d to %d", brName, gatewayIp, curMaskLen, maskLen)
		addr := fmt.Sprintf("%s/%d", gatewayIp, curMaskLen)
		log.Infof("%s: delete addr %s", logPrefix, addr)
		if err := iproute2.NewAddress(n.BridgeDev.Bridge(), addr).Del().Err(); err != nil {
			log.Warningf("%s: delete addr %s: %v", logPrefix, addr, err)
		}
		curMaskLenInt := curMaskLen
		if err := n.deleteMasqueradeRule(ctx, gatewayIp, byte(curMaskLenInt), extInterfaces); err != nil {
			log.Warningf("%s: delete iptables masqueradeRule: %v", logPrefix, err)
		}
	}
	return nil
}

type IptablesAction int

const (
	IPTABLES_ACTION_APPEND IptablesAction = iota
	IPTABLES_ACTION_DELETE
)

func (n *SNIC) doMasqueradeRule(ctx context.Context, ipStr string, maskLen byte, action IptablesAction, bridge string) error {
	gwip, err := netutils.NewIPV4Addr(ipStr)
	if err != nil {
		return errors.Wrapf(err, "NewIPV4Addr %s", ipStr)
	}
	netip := gwip.NetAddr(int8(maskLen))
	maskip := netutils.Masklen2Mask(int8(maskLen))
	var actionOpt string
	var actionInfo string
	switch action {
	case IPTABLES_ACTION_APPEND:
		actionOpt = "-A"
		actionInfo = "append"
	case IPTABLES_ACTION_DELETE:
		actionOpt = "-D"
		actionInfo = "delete"
	default:
		return errors.Errorf("unknown action %d", action)
	}
	cmd := procutils.NewCommand("iptables", "-t", "nat", actionOpt, "POSTROUTING", "-s",
		fmt.Sprintf("%s/%s", netip.String(), maskip.String()), "-o", bridge, "-j", "MASQUERADE")
	if err := cmd.Run(); err != nil {
		return errors.Wrapf(err, "%s masquerade rule", actionInfo)
	}
	return nil
}

func (n *SNIC) setupMasqueradeRule(ctx context.Context, ipStr string, maskLen byte, extInterfaces []string) error {
	if n.IsHostLocal() {
		for _, inf := range extInterfaces {
			if err := n.doMasqueradeRule(ctx, ipStr, maskLen, IPTABLES_ACTION_APPEND, inf); err != nil {
				return errors.Wrapf(err, "setupMasqueradeRule %s %d %s", ipStr, maskLen, inf)
			}
		}
		return nil
	} else {
		return n.doMasqueradeRule(ctx, ipStr, maskLen, IPTABLES_ACTION_APPEND, n.Bridge)
	}
}

func (n *SNIC) deleteMasqueradeRule(ctx context.Context, ipStr string, maskLen byte, extInterfaces []string) error {
	if n.IsHostLocal() {
		for _, inf := range extInterfaces {
			if err := n.doMasqueradeRule(ctx, ipStr, maskLen, IPTABLES_ACTION_DELETE, inf); err != nil {
				return errors.Wrapf(err, "deleteMasqueradeRule %s %d %s", ipStr, maskLen, inf)
			}
		}
		return nil
	} else {
		return n.doMasqueradeRule(ctx, ipStr, maskLen, IPTABLES_ACTION_DELETE, n.Bridge)
	}
}

func (n *SNIC) fetchHostLocalNetworks(ctx context.Context) ([]computeapis.NetworkDetails, error) {
	return fetchHostLocalNetworksByWireId(ctx, n.WireId)
}

func fetchHostLocalNetworksByWireId(ctx context.Context, wireId string) ([]computeapis.NetworkDetails, error) {
	s := hostutils.GetComputeSession(ctx)
	params := computeoptions.NetworkListOptions{}
	params.ServerType = "hostlocal"
	limit := 50
	params.Limit = &limit
	params.Scope = "system"
	total := -1
	nets := make([]computeapis.NetworkDetails, 0)
	for total < 0 || len(nets) < total {
		offset := len(nets)
		params.Offset = &offset
		results, err := computemodules.Networks.ListInContext(s, jsonutils.Marshal(params), &computemodules.Wires, wireId)
		if err != nil {
			return nil, errors.Wrap(err, "Networks.List")
		}
		total = results.Total
		for i := range results.Data {
			netDetails := computeapis.NetworkDetails{}
			err := results.Data[i].Unmarshal(&netDetails)
			if err != nil {
				return nil, errors.Wrap(err, "Unmarshal")
			}
			nets = append(nets, netDetails)
		}
	}
	return nets, nil
}