Cloudpods/backend/pkg/hostman/guestman/kickstart_helper.go

// Copyright 2019 Yunion
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package guestman

import (
	"bufio"
	"context"
	"fmt"
	"io"
	"net/http"
	"os"
	"path"
	"path/filepath"
	"regexp"
	"runtime/debug"
	"strings"
	"time"

	"gopkg.in/yaml.v3"

	"yunion.io/x/jsonutils"
	"yunion.io/x/log"
	"yunion.io/x/pkg/errors"

	api "yunion.io/x/onecloud/pkg/apis/compute"
	"yunion.io/x/onecloud/pkg/hostman/hostutils"
	modules "yunion.io/x/onecloud/pkg/mcclient/modules/compute"
	"yunion.io/x/onecloud/pkg/util/fileutils2"
	"yunion.io/x/onecloud/pkg/util/mountutils"
	"yunion.io/x/onecloud/pkg/util/procutils"
)

const (
	KICKSTART_MONITOR_TIMEOUT         = 60 * time.Minute
	SERIAL_FILE_CHECK_INTERVAL        = 5 * time.Second
	KICKSTART_ISO_MOUNT_DIR           = "iso-mount"
	KICKSTART_ISO_BUILD_DIR           = "iso-build"
	KICKSTART_ISO_FILENAME            = "config.iso"
	REDHAT_KICKSTART_ISO_VOLUME_LABEL = "OEMDRV"
	UBUNTU_KICKSTART_ISO_VOLUME_LABEL = "CIDATA"
)

var (
	kickstartInstallingRegex = regexp.MustCompile(`(?i).*KICKSTART_INSTALLING.*`)
	kickstartCompletedRegex  = regexp.MustCompile(`(?i).*KICKSTART_COMPLETED.*`)
	kickstartFailedRegex     = regexp.MustCompile(`(?i).*KICKSTART_FAILED.*`)
)

// ensureKickstartCompletionSignal ensures that the kickstart config contains the completion signal
// 1. If already contains completion signal, return as is
// 1. If no %post section, append one with completion signal
// 2. If %post exists but no %end, append completion signal before end of file
// 3. If both %post and %end exist, insert completion signal before %end
func ensureKickstartCompletionSignal(config string) string {
	completionSignals := []string{
		"echo \"KICKSTART_COMPLETED\" > /dev/ttyS1",
		"echo 'KICKSTART_COMPLETED' > /dev/ttyS1",
		"echo KICKSTART_COMPLETED > /dev/ttyS1",
	}

	for _, signal := range completionSignals {
		if strings.Contains(config, signal) {
			return config
		}
	}

	const standardCompletionSignal = "echo \"KICKSTART_COMPLETED\" > /dev/ttyS1"

	postIdx := strings.Index(config, "%post")
	if postIdx == -1 {
		return config + fmt.Sprintf("\n\n%%post\n%s\n%%end", standardCompletionSignal)
	}

	endIdx := strings.Index(config[postIdx:], "%end")
	if endIdx == -1 {
		return config + fmt.Sprintf("\n%s\n%%end", standardCompletionSignal)
	}

	// Find line index of %end within %post section
	lines := strings.Split(config, "\n")
	absoluteEndIdx := postIdx + endIdx
	endLineIdx := -1
	currentPos := 0

	for i, line := range lines {
		lineEndPos := currentPos + len(line) + 1
		if currentPos <= absoluteEndIdx && absoluteEndIdx < lineEndPos {
			if strings.TrimSpace(line) == "%end" {
				endLineIdx = i
				break
			}
		}
		currentPos = lineEndPos
	}

	if endLineIdx != -1 {
		newLines := make([]string, len(lines)+1)
		copy(newLines, lines[:endLineIdx])
		newLines[endLineIdx] = standardCompletionSignal
		copy(newLines[endLineIdx+1:], lines[endLineIdx:])
		return strings.Join(newLines, "\n")
	}

	return config
}

// ensureAutoinstallCompletionSignal ensures that the autoinstall config contains the completion signal
// 1. If already contains completion signal, return as is
// 2. If no late-commands section, create one with completion signal
// 3. If late-commands exists but no completion signal, append completion signal
func ensureAutoinstallCompletionSignal(config string) string {
	const standardCompletionSignal = "echo \"KICKSTART_COMPLETED\" > /dev/ttyS1"

	completionSignals := []string{
		"echo \"KICKSTART_COMPLETED\" > /dev/ttyS1",
		"echo 'KICKSTART_COMPLETED' > /dev/ttyS1",
		"echo KICKSTART_COMPLETED > /dev/ttyS1",
	}

	// Check if completion signal already exists
	for _, signal := range completionSignals {
		if strings.Contains(config, signal) {
			return config
		}
	}

	// Parse YAML using string keys, use yaml.v3 for better compatibility
	var yamlData map[string]interface{}
	err := yaml.Unmarshal([]byte(config), &yamlData)
	if err != nil {
		log.Warningf("Failed to parse autoinstall config as YAML: %v\n", err)
		return config
	}

	autoinstall, ok := yamlData["autoinstall"].(map[string]interface{})
	if !ok {
		log.Warningf("No autoinstall section found in config\n")
		return config
	}

	lateCommands, exists := autoinstall["late-commands"]
	if !exists {
		autoinstall["late-commands"] = []interface{}{standardCompletionSignal}
	} else {
		if commands, ok := lateCommands.([]interface{}); ok {
			autoinstall["late-commands"] = append(commands, standardCompletionSignal)
		} else {
			autoinstall["late-commands"] = []interface{}{standardCompletionSignal}
		}
	}

	result, err := yaml.Marshal(yamlData)
	if err != nil {
		log.Warningf("Failed to marshal autoinstall config as YAML: %v\n", err)
		return config
	}

	// #cloud-config is required at the top if originally present
	return "#cloud-config\n" + string(result)
}

// ensureCompletionSignal ensures that the configuration contains the completion signal based on OS type
func ensureCompletionSignal(osType, config string) string {
	switch osType {
	case "centos", "rhel", "fedora", "openeuler":
		return ensureKickstartCompletionSignal(config)
	case "ubuntu":
		return ensureAutoinstallCompletionSignal(config)
	default:
		return config // No validation for unsupported OS types
	}
}

type SKickstartSerialMonitor struct {
	serverId        string
	serialFilePath  string
	kickstartTmpDir string

	ctx    context.Context
	cancel context.CancelFunc
}

// NewKickstartSerialMonitor creates a new kickstart serial monitor
func NewKickstartSerialMonitor(s *SKVMGuestInstance) *SKickstartSerialMonitor {
	ctx, cancel := context.WithCancel(context.Background())

	// Get the server instance to access its homeDir
	serialFilePath := path.Join(s.HomeDir(), "kickstart-serial.log")
	kickstartTmpDir := s.getKickstartTmpDir()

	return &SKickstartSerialMonitor{
		serverId:        s.GetId(),
		serialFilePath:  serialFilePath,
		kickstartTmpDir: kickstartTmpDir,
		ctx:             ctx,
		cancel:          cancel,
	}
}

func (m *SKickstartSerialMonitor) GetSerialFilePath() string {
	return m.serialFilePath
}

// getKickstartTimeout gets kickstart timeout from kickstart config, defaults to KICKSTART_MONITOR_TIMEOUT
func (m *SKickstartSerialMonitor) getKickstartTimeout() time.Duration {
	server, exists := guestManager.GetServer(m.serverId)
	if !exists {
		return KICKSTART_MONITOR_TIMEOUT
	}

	kvmGuest, ok := server.(*SKVMGuestInstance)
	if !ok {
		return KICKSTART_MONITOR_TIMEOUT
	}

	kickstartConfigStr, exists := kvmGuest.Desc.Metadata[api.VM_METADATA_KICKSTART_CONFIG]
	if !exists || kickstartConfigStr == "" {
		return KICKSTART_MONITOR_TIMEOUT
	}

	configObj, err := jsonutils.ParseString(kickstartConfigStr)
	if err != nil {
		return KICKSTART_MONITOR_TIMEOUT
	}

	var config api.KickstartConfig
	if err := configObj.Unmarshal(&config); err != nil {
		return KICKSTART_MONITOR_TIMEOUT
	}

	if config.TimeoutMinutes <= 0 {
		return KICKSTART_MONITOR_TIMEOUT
	}

	timeout := time.Duration(config.TimeoutMinutes) * time.Minute
	log.Debugf("Using kickstart timeout %v for server %s", timeout, m.serverId)
	return timeout
}

func (m *SKickstartSerialMonitor) ensureSerialFile() error {
	if _, err := os.Stat(m.serialFilePath); os.IsNotExist(err) {
		file, err := os.Create(m.serialFilePath)
		if err != nil {
			return errors.Wrapf(err, "create serial file %s", m.serialFilePath)
		}
		file.Close()
		log.Debugf("Created kickstart serial file %s for server %s", m.serialFilePath, m.serverId)
	}
	return nil
}

func (m *SKickstartSerialMonitor) monitorSerialFile() {
	defer func() {
		if r := recover(); r != nil {
			log.Errorf("KickstartSerialMonitor monitor %v %s", r, debug.Stack())
		}
	}()

	var lastSize int64 = 0

	ticker := time.NewTicker(SERIAL_FILE_CHECK_INTERVAL)
	defer ticker.Stop()

	for {
		select {
		case <-m.ctx.Done():
			return
		case <-ticker.C:
			if err := m.scanSerialForStatus(&lastSize); err != nil {
				log.Errorf("Failed to scan serial file for status for server %s: %v", m.serverId, err)
			}
		}
	}
}

// scanSerialForStatus scans the serial file for a kickstart status update.
// It reads new content since the last check, parses it for status keywords,
// and triggers status updates and cleanup when a final status is detected.
func (m *SKickstartSerialMonitor) scanSerialForStatus(lastSize *int64) error {
	fileInfo, err := os.Stat(m.serialFilePath)
	if os.IsNotExist(err) {
		return nil
	}
	if err != nil {
		return err
	}

	currentSize := fileInfo.Size()
	if currentSize <= *lastSize {
		return nil
	}

	// Read new content
	file, err := os.Open(m.serialFilePath)
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = file.Seek(*lastSize, 0)
	if err != nil {
		return err
	}

	scanner := bufio.NewScanner(file)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		if len(line) == 0 {
			continue
		}

		var status string
		var shouldClose bool = false
		var matched bool = false

		if kickstartInstallingRegex.MatchString(line) {
			status = api.VM_KICKSTART_INSTALLING
			shouldClose = false
			matched = true
		} else if kickstartCompletedRegex.MatchString(line) {
			status = api.VM_KICKSTART_COMPLETED
			shouldClose = true
			matched = true
			// Unmount ISO and restart VM if kickstart install successfully
			if err := m.handleKickstartCompleted(); err != nil {
				log.Errorf("Failed to handle kickstart success for server %s: %v", m.serverId, err)
			}
		} else if kickstartFailedRegex.MatchString(line) {
			// TODO: auto retry or alert and stop
			status = api.VM_KICKSTART_FAILED
			shouldClose = true
			matched = true
		}

		if matched {
			log.Infof("Kickstart status update for server %s: %s", m.serverId, status)

			if err := m.updateKickstartStatus(status); err != nil {
				log.Errorf("Failed to update kickstart status for server %s: %v", m.serverId, err)
			} else {
				if shouldClose {
					m.Close()
					return nil
				}
			}
		}
	}

	*lastSize = currentSize
	return scanner.Err()
}

func (m *SKickstartSerialMonitor) Close() error {
	if m.cancel != nil {
		m.cancel()
	}

	if m.serialFilePath != "" {
		if err := os.Remove(m.serialFilePath); err != nil && !os.IsNotExist(err) {
			log.Warningf("Failed to remove serial file %s: %v", m.serialFilePath, err)
		}
	}

	log.Debugf("Kickstart monitor closed for server %s", m.serverId)
	return nil
}

// updateKickstartStatus updates the kickstart status via Region API
func (m *SKickstartSerialMonitor) updateKickstartStatus(status string) error {
	ctx := context.Background()
	session := hostutils.GetComputeSession(ctx)

	input := api.ServerUpdateKickstartStatusInput{
		Status: status,
	}

	_, err := modules.Servers.PerformAction(session, m.serverId, "update-kickstart-status", jsonutils.Marshal(input))
	if err != nil {
		return errors.Wrapf(err, "failed to update kickstart status for server %s", m.serverId)
	}
	return nil
}

// Start starts the kickstart serial monitor
func (m *SKickstartSerialMonitor) Start() error {
	log.Debugf("Starting kickstart monitor for server %s, serial file: %s", m.serverId, m.serialFilePath)

	if err := m.ensureSerialFile(); err != nil {
		return errors.Wrap(err, "ensure serial file")
	}

	go m.monitorSerialFile()

	// Setup timeout handler
	go func() {
		timeout := m.getKickstartTimeout()
		timer := time.NewTimer(timeout)
		defer timer.Stop()

		select {
		case <-m.ctx.Done():
			return
		case <-timer.C:
			log.Warningf("Kickstart monitor timeout (%v) for server %s, setting status to failed", timeout, m.serverId)
			if err := m.updateKickstartStatus(api.VM_KICKSTART_FAILED); err != nil {
				log.Errorf("Failed to update kickstart status to failed on timeout for server %s: %v", m.serverId, err)
			}
			m.Close()
		}
	}()

	return nil
}

// handleKickstartCompleted handles the successful kickstart completion
// It unmounts the ISO image and restarts the VM
func (m *SKickstartSerialMonitor) handleKickstartCompleted() error {
	// Clean up kickstart files after successful installation
	CleanupKickstartFiles(m.serverId, m.kickstartTmpDir)

	log.Debugf("Restarting VM %s after successful kickstart", m.serverId)

	if err := m.restartServer(); err != nil {
		return errors.Wrapf(err, "failed to restart server %s", m.serverId)
	}

	return nil
}

// restartServer restarts the server using Region API,
// because the kickstart process requires a fully reboot
// to regenerate qemu parameters
func (m *SKickstartSerialMonitor) restartServer() error {
	ctx := context.Background()
	session := hostutils.GetComputeSession(ctx)

	input := jsonutils.NewDict()
	input.Set("is_force", jsonutils.JSONFalse)

	_, err := modules.Servers.PerformAction(session, m.serverId, "restart", input)
	if err != nil {
		return errors.Wrapf(err, "failed to restart server %s via API", m.serverId)
	}

	log.Infof("Server %s restarted after kickstart completion", m.serverId)
	return nil
}

// downloadKickstartConfigFromURL downloads kickstart configuration content from the given URL
func downloadKickstartConfigFromURL(configURL string, osType string) (string, error) {
	const downloadTimeout = 10 * time.Second
	const maxConfigSize = 64 * 1024

	// For Ubuntu systems, append "/user-data" to the URL
	if osType == "ubuntu" && !strings.HasSuffix(configURL, "/user-data") {
		configURL = configURL + "/user-data"
	}

	client := &http.Client{Timeout: downloadTimeout}

	req, err := http.NewRequest("GET", configURL, nil)
	if err != nil {
		return "", errors.Wrapf(err, "failed to create download request for %s", configURL)
	}

	resp, err := client.Do(req)
	if err != nil {
		return "", errors.Wrapf(err, "failed to download config from %s", configURL)
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		return "", errors.Errorf("download failed: URL returned status code: %d", resp.StatusCode)
	}

	limitedReader := io.LimitReader(resp.Body, int64(maxConfigSize)+1)
	content, err := io.ReadAll(limitedReader)
	if err != nil {
		return "", errors.Wrapf(err, "failed to read config content from %s", configURL)
	}

	if len(content) > maxConfigSize {
		return "", errors.Errorf("downloaded content too large: %d bytes, maximum allowed: %d bytes", len(content), maxConfigSize)
	}

	if len(strings.TrimSpace(string(content))) == 0 {
		return "", errors.Error("downloaded config content is empty")
	}

	log.Debugf("Successfully downloaded kickstart config from %s: %d bytes", configURL, len(content))
	return string(content), nil
}

// CreateKickstartConfigISO creates an ISO image containing kickstart configuration files
// For Red Hat systems: creates ks.cfg in a ISO with label 'OEMDRV'
// For Ubuntu systems: creates user-data and meta-data files in a ISO with label 'CIDATA'
// reference: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/10/html/automatically_installing_rhel/starting-kickstart-installations#starting-a-kickstart-installation-automatically-using-a-local-volume
// https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/10/html/automatically_installing_rhel/starting-kickstart-installations#starting-a-kickstart-installation-automatically-using-a-local-volume
func CreateKickstartConfigISO(config *api.KickstartConfig, kickStartBaseDir string) (string, error) {
	log.Debugf("Creating kickstart ISO to %s with OS type %s", kickStartBaseDir, config.OSType)

	if config == nil {
		return "", errors.Errorf("kickstart config is nil")
	}

	// Download config content from URL if necessary
	if config.Config == "" && config.ConfigURL != "" {
		log.Debugf("Downloading kickstart config from URL: %s", config.ConfigURL)
		downloadedContent, err := downloadKickstartConfigFromURL(config.ConfigURL, config.OSType)
		if err != nil {
			return "", errors.Wrapf(err, "failed to download kickstart config from URL")
		}
		config.Config = downloadedContent
		log.Debugf("Kickstart config downloaded and set, length: %d", len(config.Config))
	}

	if config.Config == "" {
		return "", errors.Errorf("kickstart config content is empty")
	}

	// Ensure completion signal is present
	validatedConfig := ensureCompletionSignal(config.OSType, config.Config)

	// Create temporary directory for ISO contents
	tmpDir := filepath.Join(kickStartBaseDir, KICKSTART_ISO_BUILD_DIR)
	if err := os.MkdirAll(tmpDir, 0755); err != nil {
		return "", errors.Wrapf(err, "failed to create temp directory %s", tmpDir)
	}
	defer func() {
		if err := os.RemoveAll(tmpDir); err != nil {
			log.Warningf("Failed to cleanup temp directory %s: %v", tmpDir, err)
		}
	}()

	var filePaths []string
	var volumeLabel string

	switch config.OSType {
	case "centos", "rhel", "fedora", "openeuler":
		// Create anaconda-ks.cfg for Red Hat systems
		ksFilePath := filepath.Join(tmpDir, "anaconda-ks.cfg")
		if err := os.WriteFile(ksFilePath, []byte(validatedConfig), 0644); err != nil {
			return "", errors.Wrapf(err, "failed to write kickstart file %s", ksFilePath)
		}
		filePaths = []string{ksFilePath}
		volumeLabel = REDHAT_KICKSTART_ISO_VOLUME_LABEL

	case "ubuntu":
		// Create user-data file for Ubuntu systems
		userDataPath := filepath.Join(tmpDir, "user-data")
		if err := os.WriteFile(userDataPath, []byte(validatedConfig), 0644); err != nil {
			return "", errors.Wrapf(err, "failed to write user-data file %s", userDataPath)
		}

		// Create empty meta-data file
		metaDataPath := filepath.Join(tmpDir, "meta-data")
		if err := os.WriteFile(metaDataPath, []byte(""), 0644); err != nil {
			return "", errors.Wrapf(err, "failed to write meta-data file %s", metaDataPath)
		}
		filePaths = []string{userDataPath, metaDataPath}
		volumeLabel = UBUNTU_KICKSTART_ISO_VOLUME_LABEL

	default:
		return "", errors.Errorf("unsupported OS type: %s", config.OSType)
	}

	// Create ISO using mkisofs
	isoPath := filepath.Join(kickStartBaseDir, KICKSTART_ISO_FILENAME)
	args := []string{
		"-o", isoPath,
		"-V", volumeLabel,
		"-r",
		"-J",
	}
	args = append(args, filePaths...)

	cmd := procutils.NewCommand("mkisofs", args...)
	if output, err := cmd.Output(); err != nil {
		return "", errors.Wrapf(err, "mkisofs failed: %s", string(output))
	}

	log.Debugf("Successfully created kickstart ISO: %s", isoPath)
	return isoPath, nil
}

// ComputeKickstartKernelInitrdPaths returns absolute kernel and initrd paths by combining
// mountPath with OS-specific relative paths and validating that files exist.
// GetKernelInitrdPaths resolves absolute kernel and initrd paths under a mounted ISO.
func GetKernelInitrdPaths(mountPath, osType string) (string, string, error) {
	var kernelRelPath, initrdRelPath string
	switch osType {
	case "centos", "rhel", "fedora", "openeuler":
		kernelRelPath = "images/pxeboot/vmlinuz"
		initrdRelPath = "images/pxeboot/initrd.img"
	case "ubuntu":
		kernelRelPath = "casper/vmlinuz"
		initrdRelPath = "casper/initrd"
	default:
		return "", "", errors.Errorf("unsupported OS type: %s", osType)
	}

	kernelPath := path.Join(mountPath, kernelRelPath)
	initrdPath := path.Join(mountPath, initrdRelPath)

	if !fileutils2.Exists(kernelPath) {
		return "", "", errors.Errorf("kernel file not found: %s", kernelPath)
	}
	if !fileutils2.Exists(initrdPath) {
		return "", "", errors.Errorf("initrd file not found: %s", initrdPath)
	}
	return kernelPath, initrdPath, nil
}

// BuildKickstartAppendArgs builds kernel append args for kickstart/autoinstall
// based on OS type and whether a local config ISO is present.
// isoPath non-empty indicates a locally attached config ISO.
func BuildKickstartAppendArgs(config *api.KickstartConfig, isoPath string) string {
	if config == nil {
		return ""
	}
	baseArgs := []string{}
	var kickstartArgs []string
	switch config.OSType {
	case "centos", "rhel", "fedora", "openeuler":
		if isoPath != "" {
			kickstartArgs = append(kickstartArgs, fmt.Sprintf("inst.ks=hd:LABEL=%s:/anaconda-ks.cfg", REDHAT_KICKSTART_ISO_VOLUME_LABEL))
		} else if config.ConfigURL != "" {
			// Fallback to URL directly if no local ISO available
			kickstartArgs = append(kickstartArgs, fmt.Sprintf("inst.ks=%s", config.ConfigURL))
		} else {
			kickstartArgs = append(kickstartArgs, "inst.ks=cdrom:/anaconda-ks.cfg")
		}
	case "ubuntu":
		if isoPath != "" {
			kickstartArgs = append(kickstartArgs, "autoinstall")
		} else if config.ConfigURL != "" {
			// Fallback to URL directly if no local ISO available
			kickstartArgs = append(kickstartArgs,
				"autoinstall",
				"ip=dhcp",
				fmt.Sprintf("ds=nocloud-net;s=%s", config.ConfigURL),
			)
		} else {
			kickstartArgs = append(kickstartArgs, "autoinstall", "ip=dhcp", "ds=nocloud;s=/cdrom/")
		}
	}
	return strings.Join(append(baseArgs, kickstartArgs...), " ")
}

// CleanupKickstartFiles cleans up all kickstart-related temporary files and directories
func CleanupKickstartFiles(serverId, kickstartDir string) {
	if serverId == "" {
		log.Warningf("Empty serverId provided for kickstart cleanup")
		return
	}

	mountPoint := filepath.Join(kickstartDir, KICKSTART_ISO_MOUNT_DIR)
	if fileutils2.Exists(mountPoint) {
		log.Infof("Unmounting kickstart ISO at %s for server %s", mountPoint, serverId)
		if err := mountutils.Unmount(mountPoint, true); err != nil {
			log.Errorf("Failed to unmount kickstart ISO at %s: %v", mountPoint, err)
		} else {
			log.Debugf("Successfully unmounted kickstart ISO at %s for server %s", mountPoint, serverId)
		}
	}

	if server, exists := guestManager.GetServer(serverId); exists {
		if kvmGuest, ok := server.(*SKVMGuestInstance); ok && len(kvmGuest.Desc.Cdroms) > 0 {
			for _, cdrom := range kvmGuest.Desc.Cdroms {
				if cdrom.Path != "" {
					filename := path.Base(cdrom.Path)
					if strings.HasPrefix(filename, "kickstart-") {
						if err := os.Remove(cdrom.Path); err != nil && !os.IsNotExist(err) {
							log.Errorf("Failed to cleanup kickstart ISO %s: %v", cdrom.Path, err)
						} else {
							log.Debugf("Successfully removed kickstart ISO %s for server %s", cdrom.Path, serverId)
						}
						break
					}
				}
			}
		}
	}

	if fileutils2.Exists(kickstartDir) {
		log.Infof("Removing kickstart directory %s for server %s", kickstartDir, serverId)
		if err := os.RemoveAll(kickstartDir); err != nil {
			log.Errorf("Failed to remove kickstart directory %s: %v", kickstartDir, err)
		} else {
			log.Debugf("Successfully removed kickstart directory %s for server %s", kickstartDir, serverId)
		}
	}

	log.Debugf("Kickstart files cleanup completed for server %s", serverId)
}