Cloudpods/backend/pkg/util/k8s/tokens/tokens.go

// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package tokens

import (
	"context"
	"encoding/base64"
	"strings"
	"time"

	v1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/client-go/kubernetes"
	corev1 "k8s.io/client-go/kubernetes/typed/core/v1"
	"k8s.io/client-go/rest"
	bootstrapapi "k8s.io/cluster-bootstrap/token/api"
	bootstraputil "k8s.io/cluster-bootstrap/token/util"

	"yunion.io/x/jsonutils"
	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"
	"yunion.io/x/pkg/util/wait"
	"yunion.io/x/pkg/utils"

	"yunion.io/x/onecloud/pkg/util/k8s/kubeadm"
)

func GetClusterConfig() (*rest.Config, error) {
	// Load kubernetes config inside cluster
	cfg, err := rest.InClusterConfig()
	if err != nil {
		return nil, errors.Wrap(err, "get kubernetes config inside cluster")
	}
	return cfg, nil
}

func GetClient() (kubernetes.Interface, error) {
	cfg, err := GetClusterConfig()
	if err != nil {
		return nil, err
	}
	return kubernetes.NewForConfig(cfg)
}

func GetCoreClient() (corev1.CoreV1Interface, error) {
	cli, err := GetClient()
	if err != nil {
		return nil, err
	}
	return cli.CoreV1(), nil
}

func IsInsideKubernetesCluster() (bool, error) {
	_, err := GetCoreClient()
	if err != nil {
		return false, err
	}
	return true, nil
}

func GetControlPlaneEndpoint() (string, error) {
	coreCli, err := GetCoreClient()
	if err != nil {
		return "", errors.Wrap(err, "get cluster control plane endpoint")
	}
	configMap, err := coreCli.ConfigMaps(metav1.NamespaceSystem).Get(context.Background(), kubeadm.KubeadmConfigConfigMap, metav1.GetOptions{})
	if err != nil {
		return "", errors.Wrap(err, "get kubeadm cluster config")
	}
	clusterConfig, err := kubeadm.GetClusterConfigurationFromConfigMap(configMap)
	if err != nil {
		return "", errors.Wrap(err, "get kubeadm cluster configuration")
	}
	return clusterConfig.ControlPlaneEndpoint, nil
}

func GetNodeJoinToken() (string, error) {
	coreCli, err := GetCoreClient()
	if err != nil {
		return "", errors.Wrap(err, "get node join token")
	}

	bootstrapToken, err := NewBootstrap(coreCli, 24*time.Hour)
	if err != nil {
		return "", errors.Wrap(err, "failed to create new bootstrap token")
	}
	return bootstrapToken, nil
}

func GetImageRegistries() ([]string, error) {
	cli, err := GetClient()
	if err != nil {
		return nil, errors.Wrap(err, "get k8s client")
	}
	nodes, err := cli.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{})
	if err != nil {
		return nil, errors.Wrap(err, "get k8s nodes")
	}
	masterNodes := make([]*v1.Node, 0)
	for _, n := range nodes.Items {
		if _, ok := n.Labels["node-role.kubernetes.io/master"]; ok {
			masterNodes = append(masterNodes, &n)
		}
	}
	if len(masterNodes) == 0 {
		return nil, errors.Wrap(err, "not found master nodes")
	}
	regs := make([]string, 0)
	getImg := func(img v1.ContainerImage) {
		for _, name := range img.Names {
			parts := strings.Split(name, "/")
			if len(parts) == 0 {
				continue
			}
			imgRepo := parts[0]
			if !strings.Contains(imgRepo, ".") {
				// filter image like: grafana, nginx
				continue
			}
			if utils.IsInStringArray(imgRepo, regs) {
				continue
			}
			regs = append(regs, imgRepo)
		}
	}
	for _, n := range masterNodes {
		for _, img := range n.Status.Images {
			getImg(img)
		}
	}
	return regs, nil
}

// TODO: move to other packages
type DockerDaemonConfig struct {
	Bridge             string            `json:"bridge"`
	Iptables           bool              `json:"iptables"`
	ExecOpts           []string          `json:"exec-opts"`
	DataRoot           string            `json:"data-root"`
	LogDriver          string            `json:"log-driver"`
	LogOpts            map[string]string `json:"log-opts"`
	RegistryMirrors    []string          `json:"registry-mirrors"`
	InsecureRegistries []string          `json:"insecure-registries"`
	LiveRestore        bool              `json:"live-restore"`
}

func GetDockerDaemonConfig() (*DockerDaemonConfig, error) {
	regs, err := GetImageRegistries()
	if err != nil {
		return nil, err
	}
	return &DockerDaemonConfig{
		Bridge:    "none",
		Iptables:  false,
		ExecOpts:  []string{"native.cgroupdriver=systemd"},
		DataRoot:  "/opt/docker",
		LogDriver: "json-file",
		LogOpts: map[string]string{
			"max-size": "100m",
		},
		InsecureRegistries: regs,
		LiveRestore:        true,
	}, nil
}

func GetDockerDaemonContent() (string, error) {
	cfg, err := GetDockerDaemonConfig()
	if err != nil {
		return "", nil
	}
	content := jsonutils.Marshal(cfg).PrettyString()
	return base64.StdEncoding.EncodeToString([]byte(content)), nil
}

var (
	MaximumRetries = 5
)

// NewBootstrap attempts to create a token with the given ID.
func NewBootstrap(client corev1.SecretsGetter, ttl time.Duration) (string, error) {
	token, err := bootstraputil.GenerateBootstrapToken()
	if err != nil {
		return "", errors.Wrap(err, "unable to generate bootstrap token")
	}

	substrs := bootstraputil.BootstrapTokenRegexp.FindStringSubmatch(token)
	if len(substrs) != 3 {
		return "", errors.Wrapf(err, "the bootstrap token %q was not of the form %q", token, bootstrapapi.BootstrapTokenPattern)
	}
	tokenID := substrs[1]
	tokenSecret := substrs[2]

	secretName := bootstraputil.BootstrapTokenSecretName(tokenID)
	secretToken := &v1.Secret{
		ObjectMeta: metav1.ObjectMeta{
			Name:      secretName,
			Namespace: metav1.NamespaceSystem,
		},
		Type: bootstrapapi.SecretTypeBootstrapToken,
		Data: map[string][]byte{
			bootstrapapi.BootstrapTokenIDKey:               []byte(tokenID),
			bootstrapapi.BootstrapTokenSecretKey:           []byte(tokenSecret),
			bootstrapapi.BootstrapTokenExpirationKey:       []byte(time.Now().UTC().Add(ttl).Format(time.RFC3339)),
			bootstrapapi.BootstrapTokenUsageSigningKey:     []byte("true"),
			bootstrapapi.BootstrapTokenUsageAuthentication: []byte("true"),
			bootstrapapi.BootstrapTokenExtraGroupsKey:      []byte("system:bootstrappers:kubeadm:default-node-token"),
			bootstrapapi.BootstrapTokenDescriptionKey:      []byte("Node join token generate by 'onecloud region server'"),
		},
	}

	err = TryRunCommand(func() error {
		_, err := client.Secrets(secretToken.ObjectMeta.Namespace).Create(context.Background(), secretToken, metav1.CreateOptions{})
		log.Errorf("create secrets %s/%s error: %v", secretToken.GetNamespace(), secretToken.GetName(), err)
		return err
	}, MaximumRetries)
	if err != nil {
		return "", errors.Wrap(err, "unable to create secret")
	}

	return token, nil
}

// TryRunCommand runs a function a maximum of failureThreshold times, and retries on error. If failureThreshold is hit; the last error is returned
func TryRunCommand(f func() error, failureThreshold int) error {
	backoff := wait.Backoff{
		Duration: 5 * time.Second,
		Factor:   2, // double the timeout for every failure
		Steps:    failureThreshold,
	}
	return wait.ExponentialBackoff(backoff, func() (bool, error) {
		err := f()
		if err != nil {
			// Retry until the timeout
			return false, nil
		}
		// The last f() call was a success, return cleanly
		return true, nil
	})

}