Cloudpods/backend/pkg/hostman/container/snapshot_service/snapshot_service.go

package snapshot_service

import (
	"context"
	"net"
	"os"
	"path"
	"strings"

	snapshotsapi "github.com/containerd/containerd/api/services/snapshots/v1"
	"github.com/containerd/containerd/contrib/snapshotservice"
	"github.com/containerd/containerd/mount"
	"github.com/containerd/containerd/snapshots"
	"github.com/containerd/containerd/snapshots/overlay"
	"google.golang.org/grpc"

	"yunion.io/x/jsonutils"
	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"
)

func StartService(guestMan IGuestManager, root string) error {
	sn, err := NewSnapshotter(guestMan, root)
	if err != nil {
		return errors.Wrap(err, "NewSnapshotter")
	}
	svc := snapshotservice.FromSnapshotter(sn)
	rpc := grpc.NewServer()
	snapshotsapi.RegisterSnapshotsServer(rpc, svc)
	socksPath := path.Join(root, SocksFileName)
	if err := os.RemoveAll(socksPath); err != nil {
		return errors.Wrapf(err, "RemoveAll %s", socksPath)
	}
	listener, err := net.Listen("unix", socksPath)
	if err != nil {
		return errors.Wrapf(err, "Listen %s", socksPath)
	}
	return rpc.Serve(listener)
}

type IGuestManager interface {
	GetContainerManager(serverId string) (ISnapshotContainerManager, error)
}

type ISnapshotContainerManager interface {
	GetRootFsMountPath(containerId string) (string, error)
}

func NewSnapshotter(guestMan IGuestManager, root string, opts ...overlay.Opt) (snapshots.Snapshotter, error) {
	sn, err := overlay.NewSnapshotter(root, opts...)
	if err != nil {
		return nil, errors.Wrap(err, "NewSnapshotter")
	}
	return &overlayRootFsUpperSnapshotter{
		Snapshotter: sn,
		guestMan:    guestMan,
	}, nil
}

type overlayRootFsUpperSnapshotter struct {
	snapshots.Snapshotter
	guestMan IGuestManager
}

func (s *overlayRootFsUpperSnapshotter) Mounts(ctx context.Context, key string) ([]mount.Mount, error) {
	info, err := s.Snapshotter.Stat(ctx, key)
	if err != nil {
		return nil, errors.Wrapf(err, "Stat with %s", key)
	}
	infoJson := jsonutils.Marshal(info)
	log.Infof("mount request info: %s", infoJson.String())
	mounts, err := s.Snapshotter.Mounts(ctx, key)
	if err != nil {
		return nil, errors.Wrapf(err, "Mounts with %s", key)
	}
	mounts, err = s.changeUpper(ctx, key, mounts)
	if err != nil {
		return nil, errors.Wrapf(err, "Mounts.changeUpper with %s", key)
	}
	return mounts, nil
}

func (s *overlayRootFsUpperSnapshotter) Prepare(ctx context.Context, key string, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) {
	mounts, err := s.Snapshotter.Prepare(ctx, key, parent, opts...)
	if err != nil {
		return nil, errors.Wrapf(err, "Prepare with %s", key)
	}
	infoJson := jsonutils.Marshal(mounts)
	log.Debugf("prepare request key: %s, parent: %s, mounts: %s", key, parent, infoJson.String())
	mounts, err = s.changeUpper(ctx, key, mounts)
	if err != nil {
		return nil, errors.Wrapf(err, "Prepare.changeUpper with %s", key)
	}
	return mounts, nil
}

func (s *overlayRootFsUpperSnapshotter) View(ctx context.Context, key string, parent string, opts ...snapshots.Opt) ([]mount.Mount, error) {
	mounts, err := s.Snapshotter.View(ctx, key, parent, opts...)
	if err != nil {
		return nil, errors.Wrapf(err, "View with %s", key)
	}
	infoJson := jsonutils.Marshal(mounts)
	log.Debugf("view request key: %s, parent: %s, info: %s", key, parent, infoJson.String())
	mounts, err = s.changeUpper(ctx, key, mounts)
	if err != nil {
		return nil, errors.Wrapf(err, "View.changeUpper with %s", key)
	}
	return mounts, nil
}

func (s *overlayRootFsUpperSnapshotter) changeUpper(ctx context.Context, key string, mounts []mount.Mount) ([]mount.Mount, error) {
	if len(mounts) != 1 || mounts[0].Type != "overlay" {
		return mounts, nil
	}
	info, err := s.Snapshotter.Stat(ctx, key)
	if err != nil {
		return nil, errors.Wrapf(err, "Stat with %s", key)
	}
	log.Debugf("change upper key: %s, info: %s", key, jsonutils.Marshal(info))
	serverId, ok := info.Labels[LabelServerId]
	if !ok || serverId == "" {
		return mounts, nil
	}
	containerId, ok := info.Labels[LabelContainerId]
	if !ok || containerId == "" {
		return mounts, nil
	}
	ctrMan, err := s.guestMan.GetContainerManager(serverId)
	if err != nil {
		return mounts, errors.Wrapf(err, "GetContainerManager with %s", serverId)
	}
	rootFsPath, err := ctrMan.GetRootFsMountPath(containerId)
	if err != nil {
		return mounts, errors.Wrapf(err, "GetRootFsMountPath with %s, %s", serverId, containerId)
	}
	log.Debugf("changeUpper: rootFsPath: %s , container: %s", rootFsPath, containerId)
	upperPath := path.Join(rootFsPath, "upper")
	workPath := path.Join(rootFsPath, "work")
	for _, dir := range []string{upperPath, workPath} {
		if err := os.MkdirAll(dir, 0755); err != nil {
			return mounts, errors.Wrapf(err, "MkdirAll %s", dir)
		}
	}

	upperDirKey := "upperdir="
	workDirKey := "workdir="
	for i, o := range mounts[0].Options {
		if strings.HasPrefix(o, upperDirKey) {
			mounts[0].Options[i] = upperDirKey + upperPath
		}
		if strings.HasPrefix(o, workDirKey) {
			mounts[0].Options[i] = workDirKey + workPath
		}
	}
	return mounts, nil
}