Cloudpods/ocboot/onecloud/roles/utils/auto-backup-services/files/db_backup.sh

#!/usr/bin/env bash

set -e
set -o pipefail

if [ "$DEBUG" == "true" ]; then
    set -ex
    export PS4='+(${BASH_SOURCE}:${LINENO}): ${FUNCNAME[0]:+${FUNCNAME[0]}(): }'
fi

export PATH=$PATH:/opt/yunion/bin
export KUBECONFIG=/etc/kubernetes/admin.conf
export BKUP_DATE=$(date +"%Y%m%d-%H%M%S")
export BKUP_PATH=${BKUP_PATH:-/opt/yunion/backup}/$BKUP_DATE
export DB_HOST=
export DB_USER=
export DB_PSWD=
export DB_PORT=
export MYSQL_BACKUP_ARGS='--add-drop-database --add-drop-table --add-locks --single-transaction --quick'
export MAX_BKUP=${MAX_BKUP:-10}
export LIGHT_BKUP=${LIGHT_BKUP:-true}
export VERBOSE=${VERBOSE:-false}
export PV_ARGS='pv --wait --timer --rate --eta --size '
export MAX_DISK_PERCENTAGE=${MAX_DISK_PERCENTAGE:-75}
export K3S_CMDLINE_PREFIX=

if hash k3s && [[ -s ~/.kube/config ]] &>/dev/null; then
    export KUBECONFIG=~/.kube/config
    export K3S_CMDLINE_PREFIX=k3s
fi

dbs=()

__info() {
    line="$*"
    local regex="mysqldump|mysql|hostname|Light|DBs|Size:.*B\b|Backup result|Backup Disk"
    echo "[$(date +"%F %T")] $line" | grep --no-messages --color=auto --ignore-case --perl-regexp "$regex" ||
        echo "[$(date +"%F %T")] $line"
}

convert_seconds() {
    local seconds="$1"
    local hours=$((seconds / 3600))
    local minutes=$(((seconds % 3600) / 60))
    local seconds=$((seconds % 60))

    if [ "$hours" -gt 0 ]; then
        echo "$hours hours, $minutes minutes, $seconds seconds"
    elif [ "$minutes" -gt 0 ]; then
        echo "$minutes minutes, $seconds seconds"
    else
        echo "$seconds seconds"
    fi
}

title() {
    local termwidth="$(tput cols)"
    termwidth=$((termwidth / 2))
    local padding="$(printf '%0.1s' ={1..500})"

    printf '\n%*.*s %s %*.*s\n' 0 "$(((termwidth - 2 - ${#1}) / 2))" "$padding" "$1" 0 "$(((termwidth - 1 - ${#1}) / 2))" "$padding"
}

check_bin_exist() {
    for bin in "$@"; do
        __info $($bin --version)
    done
}

check_backup_disk() {
    local disk
    if [ ! -d "$BKUP_PATH" ]; then
        mkdir -p "$BKUP_PATH"
    fi

    disk=$(df --output=source "$BKUP_PATH" | grep -v '^Filesystem' | grep -v '文件系统' | head -1)
    if [ -z "$disk" ]; then
        "get disk error!"
        exit 1
    fi

    disk_usage=$(df -h --output=pcent --exclude-type=tmpfs,devtmpfs "$disk" | grep -v 'Use%' | head -1 | tr -d % | xargs)
    if [ "$disk_usage" -gt $MAX_DISK_PERCENTAGE ]; then
        __info "$BKUP_PATH is on disk $disk, usage:$disk_usage%, too high to continue backup."
        __info "Allowed maximum disk usage: $MAX_DISK_PERCENTAGE."
        # only delete backup path when it is empty and ends with timestamp like 20240506-163855
        if [ -z "$(ls -A $BKUP_PATH)" ] && [[ "$BKUP_PATH" =~ [0-9]{8}-[0-9]{6}$ ]]; then
            rm -rf "$BKUP_PATH"
        fi
        exit 1
    fi
    __info "Backup Disk: $disk; Used: $disk_usage%"
}

init_db() {
    local
    title "Init"
    check_backup_disk
    local content="$($K3S_CMDLINE_PREFIX kubectl get onecloudcluster -n onecloud default -o yaml | grep -A 5 -w mysql)"
    export DB_HOST=$(echo "$content" | grep -w host: | awk '{print $(NF)}')
    export DB_USER=$(echo "$content" | grep -w username: | awk '{print $(NF)}')
    export DB_PSWD=$(echo "$content" | grep -w password: | awk '{print $(NF)}')
    export DB_PORT=$(echo "$content" | grep -w port: | awk '{print $(NF)}')
    export DB_PORT=${DB_PORT:-3306}
    local size
    if ! check_bin_exist mysql mysqldump; then
        __info "mysql or mysqldump is missing!"
        exit 1
    fi

    if [ -n "$DB_HOST" ] && [ -n "$DB_USER" ] && [ -n "$DB_PSWD" ]; then
        dbs=($(get_dbs))
        size=$(get_all_db_size_bytes)
        size=$(numfmt --to=iec --suffix=B "$size")
    else
        __info "init db error"
        exit 1
    fi

    local flag=Full
    if [[ "$LIGHT_BKUP" == "true" ]]; then
        flag=Light
    fi

    __info "Hostname: $(hostname), IP: $(ip route get 1 | head -1 | awk '{print $7}')"
    __info "performing $flag backup."
    __info "DBs to backup: [${dbs[*]}]"
    __info "Size: $size before compress."
}

query() {
    mysql --skip-column-names -h "$DB_HOST" -u "$DB_USER" -p"$DB_PSWD" -P "$DB_PORT" "$@"
}

get_dbs() {
    local filter
    if [ "$LIGHT_BKUP" == true ]; then
        filter='s#^\(Database\|mysql\|performance_schema\|test\|information_schema\|yunionlogger\|yunionmeter\)$##g'
    else
        filter='s#^\(Database\|mysql\|performance_schema\|test\|information_schema\)$##g'
    fi
    query -e 'show databases;' | sed -e "$filter" | sort
}

# --ignore-table=DB.table
get_ignore_tables_for_db() {
    local db="$1"
    local sql
    local tables=()
    if [ -z "$db" ]; then
        return
    fi
    sql="show tables from \`$db\` where \`Tables_in_$db\` like 'opslog%' or \`Tables_in_$db\` like 'task%';"
    tables=($(query -e "$sql"))
    for table in "${tables[@]}"; do
        echo "--ignore-table=$db.$table"
    done
}

get_all_ignored_tables() {
    local tables=()
    for db in "${dbs[@]}"; do
        tables+=($(get_ignore_tables_for_db $db))
    done
    echo "${tables[@]}"
}

backup_db() {
    local timestamp=$BKUP_DATE
    local args=()
    local output="$BKUP_PATH/onecloud.sql.$timestamp.gz"
    local startTime
    local endTime
    startTime=$(date +"%s")
    title "Backup Database"
    args=(${MYSQL_BACKUP_ARGS[@]} -h $DB_HOST -u $DB_USER -p"$DB_PSWD" -P "$DB_PORT")
    args+=(--databases ${dbs[@]})
    if [ "$LIGHT_BKUP" == true ]; then
        args+=($(get_all_ignored_tables))
    fi

    if [ -x /usr/bin/pv ]; then
        mysqldump "${args[@]}" | ${PV_ARGS[@]} $(get_all_db_size_bytes) | gzip -c >"$output"
    else
        mysqldump "${args[@]}" | gzip -c >"$output"
    fi
    endTime=$(date +"%s")
    __info "It takes $(convert_seconds $((endTime - startTime)))."
}

get_db_size_bytes() {
    local db="$1"
    local sql
    if [ -z "$db" ]; then
        return
    fi
    sql='SELECT
        sum(ROUND((DATA_LENGTH + INDEX_LENGTH )*0.85) )
        FROM information_schema.TABLES
        WHERE TABLE_SCHEMA = "'$db'" '
    if [[ "$LIGHT_BKUP" == "true" ]]; then
        sql="$sql"'AND table_name not like "opslog%"
        AND table_name not like "task%"'
    fi
    query -e "$sql" | xargs
}

get_all_db_size_bytes() {
    local sum=0
    local db
    local size
    for db in "${dbs[@]}"; do
        size=$(get_db_size_bytes $db)
        sum=$((sum + size))
    done
    echo $sum
}

rotate_bkups() {
    local old_bkups=()
    local max=$((MAX_BKUP + 1))
    (
        cd "$(dirname $BKUP_PATH)"
        if ! [ -d "$BKUP_PATH" ]; then
            __info "backup path $BKUP_PATH dose not exist!"
            exit 1
        fi
        __info "Archive Backup Files..."
        find $BKUP_PATH -type f | xargs ls -lah
        tar czvf onecloud.bkup.$BKUP_DATE.tar.gz $BKUP_DATE && rm -rf $BKUP_DATE
        echo
        __info "Backup result: "
        ls -lah $(readlink -f onecloud.bkup.$BKUP_DATE.tar.gz)
    )
    old_bkups=($(find $(dirname $BKUP_PATH) -name "onecloud*.gz" | sort --reverse | tail -n +$max))
    title "Cleaning"
    if [ "${#old_bkups[@]}" -gt 0 ]; then
        __info "cleaning old backups: ${old_bkups[*]}"
        if ! rm -f "${old_bkups[@]}"; then
            __info "rm old backup files error! ${old_bkups[*]}"
            exit 1
        fi
    fi
    title "All DONE"
}

backup_etcd() {
    title "Backup ETCD"
    local etcd_id="$(docker ps -a | grep etcd | grep Up | grep -v pause | grep 'etcd --adv' | head -1 | awk '{print $1}' || :)"
    local etcdctl=(docker exec $etcd_id /usr/local/bin/etcdctl)
    if [ -z "$etcd_id" ]; then
        echo "Docekr or ETCD is not running. ignore etcd backup"
        return
    fi

    __info "$(${etcdctl[@]} version | xargs)"

    ETCDCTL_API=3 ${etcdctl[@]} snapshot save /tmp/etcd_snapshot_$BKUP_DATE.db \
        --cacert /etc/kubernetes/pki/etcd/ca.crt \
        --cert /etc/kubernetes/pki/etcd/server.crt \
        --key /etc/kubernetes/pki/etcd/server.key

    ETCDCTL_API=3 ${etcdctl[@]} snapshot status /tmp/etcd_snapshot_$BKUP_DATE.db

    docker cp $etcd_id:/tmp/etcd_snapshot_$BKUP_DATE.db $BKUP_PATH

    docker exec $etcd_id rm -f /tmp/etcd_snapshot_$BKUP_DATE.db
}

backup_k8s() {
    title "Backup K8s"
    $K3S_CMDLINE_PREFIX kubectl -n onecloud -o yaml get deployment onecloud-operator >$BKUP_PATH/onecloud-operator.$BKUP_DATE.yml
    $K3S_CMDLINE_PREFIX kubectl -n onecloud -o yaml get oc >$BKUP_PATH/oc.$BKUP_DATE.yml
    sed -i -e '/^ *creationTimestamp:/d' -e '/^ *resourceVersion:/d' -e '/^ *selfLink:/d' -e '/^ *uid:/d' $BKUP_PATH/onecloud-operator.$BKUP_DATE.yml
    sed -i -e '/^ *creationTimestamp:/d' -e '/^ *resourceVersion:/d' -e '/^ *selfLink:/d' -e '/^ *uid:/d' $BKUP_PATH/oc.$BKUP_DATE.yml
    __info "K8s backup files:"
    find $BKUP_PATH -name '*yml' -type f | xargs ls -lah
}

main() {
    init_db
    backup_k8s
    backup_etcd
    backup_db
    rotate_bkups
}

main