Cloudpods/backend/pkg/compute/tasks/guest/guest_delete_task.go

// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package guest

import (
	"context"
	"fmt"

	"yunion.io/x/cloudmux/pkg/cloudprovider"
	"yunion.io/x/jsonutils"
	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"
	"yunion.io/x/pkg/utils"

	api "yunion.io/x/onecloud/pkg/apis/compute"
	"yunion.io/x/onecloud/pkg/cloudcommon/db"
	"yunion.io/x/onecloud/pkg/cloudcommon/db/taskman"
	"yunion.io/x/onecloud/pkg/cloudcommon/notifyclient"
	"yunion.io/x/onecloud/pkg/compute/models"
	"yunion.io/x/onecloud/pkg/compute/options"
	"yunion.io/x/onecloud/pkg/util/logclient"
)

type BaseGuestDeleteTask struct {
	SGuestBaseTask
}

var (
	STORAGEIDS = "storage_ids"
)

func init() {
	taskman.RegisterTask(BaseGuestDeleteTask{})
}

func (deleteTask *BaseGuestDeleteTask) OnInit(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
	guest := obj.(*models.SGuest)
	host, _ := guest.GetHost()
	if guest.Hypervisor == api.HYPERVISOR_BAREMETAL && host != nil && host.HostType != api.HOST_TYPE_BAREMETAL {
		// if a fake server for converted hypervisor, then just skip stop
		deleteTask.OnGuestStopComplete(ctx, guest, data)
		return
	}
	drv, err := guest.GetDriver()
	if err != nil {
		deleteTask.OnGuestStopComplete(ctx, guest, data)
		return
	}
	if len(guest.BackupHostId) > 0 {
		deleteTask.SetStage("OnMasterHostStopGuestComplete", nil)
		if err := drv.RequestStopGuestForDelete(ctx, guest, nil, deleteTask); err != nil {
			log.Errorf("RequestStopGuestForDelete fail %s", err)
			deleteTask.OnMasterHostStopGuestComplete(ctx, guest, nil)
		}
	} else {
		deleteTask.SetStage("OnGuestStopComplete", nil)
		if err := drv.RequestStopGuestForDelete(ctx, guest, nil, deleteTask); err != nil {
			log.Errorf("RequestStopGuestForDelete fail %s", err)
			deleteTask.OnGuestStopComplete(ctx, guest, nil)
		}
	}
}

func (deleteTask *BaseGuestDeleteTask) OnMasterHostStopGuestComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
	deleteTask.SetStage("OnGuestStopComplete", nil)
	host := models.HostManager.FetchHostById(guest.BackupHostId)
	drv, err := guest.GetDriver()
	if err != nil {
		deleteTask.OnGuestStopComplete(ctx, guest, nil)
		return
	}
	err = drv.RequestStopGuestForDelete(ctx, guest, host, deleteTask)
	if err != nil {
		log.Errorf("RequestStopGuestForDelete fail %s", err)
		deleteTask.OnGuestStopComplete(ctx, guest, nil)
	}
}

func (deleteTask *BaseGuestDeleteTask) OnMasterHostStopGuestCompleteFailed(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
	deleteTask.OnGuestStopComplete(ctx, guest, nil) // ignore stop error
}

func (deleteTask *BaseGuestDeleteTask) StartDeleteGuestSnapshots(ctx context.Context, guest *models.SGuest) {
	guest.StartDeleteGuestSnapshots(ctx, deleteTask.UserCred, deleteTask.GetTaskId())
}

func (deleteTask *BaseGuestDeleteTask) OnGuestStopComplete(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
	if jsonutils.QueryBoolean(deleteTask.Params, "delete_snapshots", false) {
		deleteTask.SetStage("OnStartEipDissociate", nil)
		guest.StartDeleteGuestSnapshots(ctx, deleteTask.UserCred, deleteTask.Id)
		return
	}
	deleteTask.OnStartEipDissociate(ctx, guest, data)
}

func (deleteTask *BaseGuestDeleteTask) OnGuestStopCompleteFailed(ctx context.Context, guest *models.SGuest, err jsonutils.JSONObject) {
	if len(guest.ExternalId) > 0 {
		_, e := guest.GetIVM(ctx)
		if errors.Cause(e) == cloudprovider.ErrNotFound {
			deleteTask.Params.Set("override_pending_delete", jsonutils.JSONTrue)
		}
	}
	deleteTask.OnGuestStopComplete(ctx, guest, err) // ignore stop error
}

func (deleteTask *BaseGuestDeleteTask) OnStartEipDissociateFailed(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
	log.Errorf("Delete guest snapshots faield: %s", data)
	deleteTask.OnStartEipDissociate(ctx, guest, nil)
}

func (deleteTask *BaseGuestDeleteTask) OnStartEipDissociate(ctx context.Context, guest *models.SGuest, data jsonutils.JSONObject) {
	if sourceGuestId := guest.GetMetadata(ctx, api.SERVER_META_CONVERT_FROM_ESXI, deleteTask.UserCred); len(sourceGuestId) > 0 {
		sourceGuest := models.GuestManager.FetchGuestById(sourceGuestId)
		if sourceGuest != nil &&
			sourceGuest.GetMetadata(ctx, api.SERVER_META_CONVERTED_SERVER, deleteTask.UserCred) == guest.Id {
			sourceGuest.RemoveMetadata(ctx, api.SERVER_META_CONVERTED_SERVER, deleteTask.UserCred)
			sourceGuest.StartSyncstatus(ctx, deleteTask.UserCred, "")
		}
	}
	eip, _ := guest.GetEipOrPublicIp()
	if eip != nil && eip.Mode != api.EIP_MODE_INSTANCE_PUBLICIP {
		// detach floating EIP only
		if jsonutils.QueryBoolean(deleteTask.Params, "purge", false) {
			// purge locally
			eip.Dissociate(ctx, deleteTask.UserCred)
			deleteTask.OnEipDissociateComplete(ctx, guest, nil)
		} else {
			deleteTask.SetStage("OnEipDissociateComplete", nil)
			autoDelete := jsonutils.QueryBoolean(deleteTask.GetParams(), "delete_eip", false)
			eip.StartEipDissociateTask(ctx, deleteTask.UserCred, autoDelete, deleteTask.GetTaskId())
		}
	} else {
		deleteTask.OnEipDissociateComplete(ctx, guest, nil)
	}
}

func (deleteTask *BaseGuestDeleteTask) OnEipDissociateCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) {
	guest := obj.(*models.SGuest)
	deleteTask.OnFailed(ctx, guest, err)
}

func (deleteTask *BaseGuestDeleteTask) OnEipDissociateComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
	deleteTask.SetStage("OnDiskDetachComplete", nil)
	deleteTask.OnDiskDetachComplete(ctx, obj, data)
}

// remove detachable disks
func (deleteTask *BaseGuestDeleteTask) OnDiskDetachComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
	log.Debugf("OnDiskDetachComplete")
	guest := obj.(*models.SGuest)

	guestdisksOrigin, _ := guest.GetGuestDisks()
	var guestdisks []models.SGuestdisk
	// clean dirty data
	for i := range guestdisksOrigin {
		if guestdisksOrigin[i].GetDisk() == nil {
			guestdisksOrigin[i].Detach(ctx, deleteTask.UserCred)
		} else {
			guestdisks = append(guestdisks, guestdisksOrigin[i])
		}
	}
	if len(guestdisks) == 0 {
		// on guest disks detached
		deleteTask.doClearGPUDevicesComplete(ctx, guest)
		return
	}
	// detach last detachable disk
	lastDisk := guestdisks[len(guestdisks)-1].GetDisk()
	deleteDisks := jsonutils.QueryBoolean(deleteTask.Params, "delete_disks", false)
	if deleteDisks {
		lastDisk.SetAutoDelete(lastDisk, deleteTask.GetUserCred(), true)
	}
	log.Debugf("lastDisk IsDetachable?? %v", lastDisk.IsDetachable())
	if !lastDisk.IsDetachable() {
		// no more disk need detach
		deleteTask.doClearGPUDevicesComplete(ctx, guest)
		return
	}
	purge := jsonutils.QueryBoolean(deleteTask.Params, "purge", false)
	guest.StartGuestDetachdiskTask(ctx, deleteTask.UserCred, lastDisk, true, deleteTask.GetTaskId(), purge, false)
}

func (deleteTask *BaseGuestDeleteTask) OnDiskDetachCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) {
	guest := obj.(*models.SGuest)
	deleteTask.OnFailed(ctx, guest, err)
}

// clean gpu devices
func (deleteTask *BaseGuestDeleteTask) doClearGPUDevicesComplete(ctx context.Context, guest *models.SGuest) {
	log.Debugf("doClearGPUDevicesComplete")
	models.IsolatedDeviceManager.ReleaseGPUDevicesOfGuest(ctx, guest, deleteTask.UserCred)
	if jsonutils.QueryBoolean(deleteTask.Params, "purge", false) {
		deleteTask.OnSyncConfigComplete(ctx, guest, nil)
	} else {
		deleteTask.SetStage("OnSyncConfigComplete", nil)
		guest.StartSyncTaskWithoutSyncstatus(ctx, deleteTask.UserCred, false, deleteTask.GetTaskId())
	}
}

func (deleteTask *BaseGuestDeleteTask) OnSyncConfigComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
	guest := obj.(*models.SGuest)

	// try to leave all groups
	guest.LeaveAllGroups(ctx, deleteTask.UserCred)
	// cleanup tap services and flows
	guest.CleanTapRecords(ctx, deleteTask.UserCred)

	isPurge := jsonutils.QueryBoolean(deleteTask.Params, "purge", false)
	overridePendingDelete := jsonutils.QueryBoolean(deleteTask.Params, "override_pending_delete", false)

	if options.Options.EnablePendingDelete && !isPurge && !overridePendingDelete {
		if guest.PendingDeleted {
			deleteTask.SetStageComplete(ctx, nil)
			return
		}
		log.Debugf("XXXXXXX Do guest pending delete... XXXXXXX")
		// pending detach
		guest.PendingDetachScalingGroup()
		guestStatus, _ := deleteTask.Params.GetString("guest_status")
		if !utils.IsInStringArray(guestStatus, []string{
			api.VM_SCHEDULE_FAILED, api.VM_NETWORK_FAILED,
			api.VM_CREATE_FAILED, api.VM_DEVICE_FAILED}) {
			deleteTask.StartPendingDeleteGuest(ctx, guest)
			return
		}
	}
	log.Debugf("XXXXXXX Do real delete on guest ... XXXXXXX")
	deleteTask.doStartDeleteGuest(ctx, guest)
}

func (deleteTask *BaseGuestDeleteTask) OnSyncConfigCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) {
	// guest := obj.(*models.SGuest)
	// deleteTask.OnFailed(ctx, guest, err)
	deleteTask.OnSyncConfigComplete(ctx, obj, err) // ignore sync config failed error
}

func (deleteTask *BaseGuestDeleteTask) OnGuestDeleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) {
	guest := obj.(*models.SGuest)
	deleteTask.OnFailed(ctx, guest, err)
}

func (deleteTask *BaseGuestDeleteTask) doStartDeleteGuest(ctx context.Context, obj db.IStandaloneModel) {
	guest := obj.(*models.SGuest)
	guest.SetStatus(ctx, deleteTask.UserCred, api.VM_DELETING, "delete server after stop")
	db.OpsLog.LogEvent(guest, db.ACT_DELOCATING, guest.GetShortDesc(ctx), deleteTask.UserCred)
	deleteTask.StartDeleteGuest(ctx, guest)
}

func (deleteTask *BaseGuestDeleteTask) StartPendingDeleteGuest(ctx context.Context, guest *models.SGuest) {
	guest.DoPendingDelete(ctx, deleteTask.UserCred)
	deleteTask.SetStage("OnPendingDeleteComplete", nil)
	guest.StartSyncstatus(ctx, deleteTask.UserCred, deleteTask.GetTaskId())
}

func (deleteTask *BaseGuestDeleteTask) OnPendingDeleteComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
	deleteTask.SetStageComplete(ctx, jsonutils.NewDict())
}

func (deleteTask *BaseGuestDeleteTask) OnPendingDeleteCompleteFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
	deleteTask.OnPendingDeleteComplete(ctx, obj, nil)
}

func (deleteTask *BaseGuestDeleteTask) StartDeleteGuest(ctx context.Context, guest *models.SGuest) {
	// Temporary storageids to sync capacityUsed after delete
	{
		storages, _ := guest.GetStorages()
		storageIds := make([]string, len(storages))
		for i := range storages {
			storageIds[i] = storages[i].GetId()
		}
		deleteTask.Params.Set(STORAGEIDS, jsonutils.NewStringArray(storageIds))
	}
	// No snapshot
	deleteTask.SetStage("OnGuestDetachDisksComplete", nil)
	drv, err := guest.GetDriver()
	if err != nil {
		deleteTask.OnGuestDeleteFailed(ctx, guest, jsonutils.NewString(err.Error()))
		return
	}
	drv.RequestDetachDisksFromGuestForDelete(ctx, guest, deleteTask)
}

func (deleteTask *BaseGuestDeleteTask) OnGuestDetachDisksComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
	guest := obj.(*models.SGuest)
	deleteTask.DoDeleteGuest(ctx, guest)
}

func (deleteTask *BaseGuestDeleteTask) OnGuestDetachDisksCompleteFailed(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
	deleteTask.OnGuestDeleteFailed(ctx, obj, data)
}

func (deleteTask *BaseGuestDeleteTask) DoDeleteGuest(ctx context.Context, guest *models.SGuest) {
	models.IsolatedDeviceManager.ReleaseDevicesOfGuest(ctx, guest, deleteTask.UserCred)
	host, _ := guest.GetHost()
	if guest.IsPrepaidRecycle() {
		err := host.BorrowIpAddrsFromGuest(ctx, deleteTask.UserCred, guest)
		if err != nil {
			msg := fmt.Sprintf("host.BorrowIpAddrsFromGuest fail %s", err)
			log.Errorf("%v", msg)
			deleteTask.OnGuestDeleteFailed(ctx, guest, jsonutils.NewString(msg))
			return
		}
		deleteTask.OnGuestDeleteComplete(ctx, guest, nil)
	} else if (host == nil || !host.GetEnabled()) && jsonutils.QueryBoolean(deleteTask.Params, "purge", false) {
		deleteTask.OnGuestDeleteComplete(ctx, guest, nil)
	} else {
		deleteTask.SetStage("OnGuestDeleteComplete", nil)
		guest.StartUndeployGuestTask(ctx, deleteTask.UserCred, deleteTask.GetTaskId(), "")
	}
}

func (deleteTask *BaseGuestDeleteTask) OnFailed(ctx context.Context, guest *models.SGuest, err jsonutils.JSONObject) {
	guest.SetStatus(ctx, deleteTask.UserCred, api.VM_DELETE_FAIL, err.String())
	db.OpsLog.LogEvent(guest, db.ACT_DELOCATE_FAIL, err, deleteTask.UserCred)
	logclient.AddActionLogWithStartable(deleteTask, guest, logclient.ACT_DELOCATE, err, deleteTask.UserCred, false)
	notifyclient.EventNotify(ctx, deleteTask.GetUserCred(), notifyclient.SEventNotifyParam{
		Obj:    guest,
		Action: notifyclient.ActionDelete,
		IsFail: true,
	})
	deleteTask.SetStageFailed(ctx, err)
}

func (deleteTask *BaseGuestDeleteTask) OnGuestDeleteCompleteFailed(ctx context.Context, obj db.IStandaloneModel, err jsonutils.JSONObject) {
	guest := obj.(*models.SGuest)
	deleteTask.OnFailed(ctx, guest, err)
}

func (deleteTask *BaseGuestDeleteTask) OnGuestDeleteComplete(ctx context.Context, obj db.IStandaloneModel, data jsonutils.JSONObject) {
	guest := obj.(*models.SGuest)
	guest.RevokeAllNetworkSecgroups(ctx, deleteTask.UserCred)
	guest.DetachAllNetworks(ctx, deleteTask.UserCred)
	guest.EjectAllIso(deleteTask.UserCred)
	guest.EjectAllVfd(deleteTask.UserCred)
	guest.DeleteEip(ctx, deleteTask.UserCred)
	drv, _ := guest.GetDriver()
	if drv != nil {
		drv.OnDeleteGuestFinalCleanup(ctx, guest, deleteTask.UserCred)
	}
	deleteTask.DeleteGuest(ctx, guest)
}

func (deleteTask *BaseGuestDeleteTask) DeleteGuest(ctx context.Context, guest *models.SGuest) {
	data := jsonutils.NewDict()
	data.Set("real_delete", jsonutils.JSONTrue)
	deleteTask.SetStageComplete(ctx, data)
}

func (deleteTask *BaseGuestDeleteTask) NotifyServerDeleted(ctx context.Context, guest *models.SGuest) {
	guest.EventNotify(ctx, deleteTask.UserCred, notifyclient.ActionPendingDelete)
}