770 lines
22 KiB
Bash
Executable File
770 lines
22 KiB
Bash
Executable File
#!/bin/bash
|
|
### Copyright 1999-2025. WebPros International GmbH. All rights reserved.
|
|
|
|
shopt -s nullglob
|
|
export LC_ALL="C" PATH=/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin
|
|
unset GREP_OPTIONS
|
|
umask 022
|
|
export PLESK_MYSQL_AUTH_SKIP_CONNECTION_CHECK=1
|
|
|
|
PRODUCT_ROOT_D="/opt/psa"
|
|
PRODUCT_LOGS_D="/var/log/plesk"
|
|
|
|
prog="`basename $0`"
|
|
action="$1"
|
|
|
|
ts()
|
|
{
|
|
[ "$PLESK_REPAIR_INNODB_TIMESTAMPS" = "none" ] ||
|
|
echo "[`date --rfc-3339=seconds`] "
|
|
}
|
|
|
|
# Communication protocol with repaird is:
|
|
# * emit output to stdout as plaintext;
|
|
# * stderr is not shown to user, but may be logged by 'repaird -v 3';
|
|
# * last line of output is JSON with any remaining output (see result() below);
|
|
# * exit code is 0.
|
|
|
|
info()
|
|
{
|
|
echo "`ts`INFO: $*"
|
|
}
|
|
|
|
warn()
|
|
{
|
|
echo "`ts`WARNING: $*"
|
|
}
|
|
|
|
err()
|
|
{
|
|
echo "`ts`ERROR: $*"
|
|
}
|
|
|
|
die()
|
|
{
|
|
err "$@" | result "failed"
|
|
exit 0
|
|
}
|
|
|
|
success()
|
|
{
|
|
info "$@" | result "ok"
|
|
exit 0
|
|
}
|
|
|
|
usage()
|
|
{
|
|
echo "Usage: $prog { --check | --repair }" >&2
|
|
exit 1
|
|
}
|
|
|
|
result()
|
|
{
|
|
local status="$1"
|
|
python3 -c 'import sys, json
|
|
message = sys.stdin.read()
|
|
status = sys.argv[1] if message.strip() else "ok"
|
|
print(json.dumps({"status": status, "message": message}))
|
|
' "$status"
|
|
}
|
|
|
|
log_if_failed()
|
|
{
|
|
# Emits command output to stdout if it failed, otherwise to stderr
|
|
local out=
|
|
local rc=
|
|
|
|
if out="`"$@" 2>&1`"; then
|
|
echo "$out" >&2
|
|
return 0
|
|
else
|
|
rc="$?"
|
|
echo "$out"
|
|
return "$rc"
|
|
fi
|
|
}
|
|
|
|
is_remote_db_feature_enabled()
|
|
{
|
|
[ -s "/etc/psa/private/dsn.ini" ]
|
|
}
|
|
|
|
get_psa_conf_option_value()
|
|
{
|
|
local name="$1"
|
|
local default="$2"
|
|
local prod_conf_t="/etc/psa/psa.conf"
|
|
|
|
local value="`sed -n "s/^\s*$name\s*//p" "$prod_conf_t" | tail -n 1`"
|
|
echo "${value:-$default}"
|
|
}
|
|
|
|
get_plesk_dump_d()
|
|
{
|
|
get_psa_conf_option_value "DUMP_D" "/var/lib/psa/dumps"
|
|
}
|
|
|
|
create_backup_dir()
|
|
{
|
|
local dump_d="`get_plesk_dump_d`"
|
|
local timestamp="`date +%Y%m%d-%H%M%0S`"
|
|
local backup_d="$dump_d/mysql.repair-innodb.$timestamp.d"
|
|
mkdir "$backup_d" || return 1
|
|
|
|
echo "$backup_d"
|
|
}
|
|
|
|
log_backup_dir_location()
|
|
{
|
|
local backup_d="$1"
|
|
local plesk_cron_daily="/etc/cron.daily/50plesk-daily"
|
|
local lifetime= expire_by=
|
|
lifetime="`grep -F "mysql.repair-innodb.*.d" "$plesk_cron_daily" | grep -E -o '\<[0-9]+\s+days\>'`"
|
|
[ -z "$lifetime" ] || expire_by="`date -d "$(date -r "$backup_d") +$lifetime"`"
|
|
|
|
info "Backups will be stored under '$backup_d'." \
|
|
${lifetime:+"Directory lifetime: $lifetime (will be automatically removed after $expire_by)."}
|
|
}
|
|
|
|
find_latest_preexisting_dump()
|
|
{
|
|
local dump_d="`get_plesk_dump_d`"
|
|
find "$dump_d" \
|
|
-type f \( \
|
|
-name 'mysql.preupgrade.*' -a ! -name 'mysql.preupgrade.apsc.*' -o \
|
|
-name 'mysql.daily.*' -o \
|
|
-name 'mysql.pre-rdbms-upgrade.*' \
|
|
\) \
|
|
-printf "%T@\t%p\n" | LC_ALL=C sort -nr -k1 | head -n1 | cut -f2
|
|
}
|
|
|
|
get_mysql_option_value()
|
|
{
|
|
local section="$1"
|
|
local name="$2"
|
|
local default="$3"
|
|
|
|
local value="`/usr/bin/my_print_defaults "$section" | sed -n "s/^--$name=//p" | tail -n 1`"
|
|
echo "${value:-$default}"
|
|
}
|
|
|
|
get_mysql_datadir()
|
|
{
|
|
get_mysql_option_value mysqld datadir "/var/lib/mysql"
|
|
}
|
|
|
|
mysql_action()
|
|
{
|
|
local action="$1"
|
|
|
|
if [ "$action" = "start" -o "$action" = "restart" ]; then
|
|
# The script may restart MySQL often, which may lead to it failing with 'start-limit-hit'.
|
|
[ -n "$MYSQL_SERVICE_NAME" ] || MYSQL_SERVICE_NAME="`"$PRODUCT_ROOT_D/admin/sbin/pleskrc" mysql name`"
|
|
[ -z "$MYSQL_SERVICE_NAME" ] || /bin/systemctl reset-failed "$MYSQL_SERVICE_NAME"
|
|
fi
|
|
|
|
"$PRODUCT_ROOT_D/admin/sbin/pleskrc" mysql "$1" 1>&2 || {
|
|
# 'pleskrc' output is usually not very pretty or informative, so don't show it to user directly.
|
|
local log_path="$PRODUCT_LOGS_D/rc_actions.log"
|
|
info "Failed to $action the database service. See $log_path and the service log for details."
|
|
return 1
|
|
}
|
|
}
|
|
|
|
mysql_disable_respawn()
|
|
{
|
|
[ -n "$MYSQL_SERVICE_NAME" ] || MYSQL_SERVICE_NAME="`"$PRODUCT_ROOT_D/admin/sbin/pleskrc" mysql name`"
|
|
|
|
local unit_drop_in="/lib/systemd/system/$MYSQL_SERVICE_NAME.d/respawn.conf"
|
|
[ -f "$unit_drop_in" ] || return 0
|
|
|
|
# This action is later reverted by 'mysqlmng --post-re-init', if needed. However, in practice
|
|
# it may be reverted only on CentOS 7 like systems, since other versions already include own
|
|
# Restart unit configuration (and don't need ours). This means that effective CentOS 7
|
|
# configuration may change as a result of repair, but I'm willing to make this small sacrifice.
|
|
rm -f "$unit_drop_in"
|
|
/bin/systemctl daemon-reload
|
|
info "Plesk-configured automatic restart of $MYSQL_SERVICE_NAME was temporarily disabled."
|
|
}
|
|
|
|
report_disk_space_for_repair()
|
|
{
|
|
local datadir="$1"
|
|
local dump_d="`get_plesk_dump_d`"
|
|
# This allows a very rough estimate of space that would be consumed by created dump files.
|
|
local latest_dump="`find_latest_preexisting_dump`"
|
|
local du_estimate="`du -cshD "$datadir" $latest_dump | tail -n1 | cut -f1`"
|
|
local du_mount_point="`df -h --output=target "$dump_d" | tail -n1`"
|
|
local du_available="`df -h --output=avail "$dump_d" | tail -n1 | xargs`"
|
|
|
|
info "Repair of InnoDB corruption will require at least $du_estimate of free disk space" \
|
|
"on $du_mount_point mount point. Currently $du_available of disk space is available on it."
|
|
}
|
|
|
|
check()
|
|
{
|
|
! is_remote_db_feature_enabled || die "Cannot check for InnoDB corruption on a remote database."
|
|
mysql_action "stop" || die "Could not stop the database service."
|
|
|
|
{
|
|
local rc=0
|
|
local datadir="`get_mysql_datadir`"
|
|
# Default system tablespace data file is 'ibdata1', assume others (if any) are named similarly.
|
|
# Alternatively we should parse innodb_data_file_path value, which may include many paths.
|
|
# On some versions (e.g. Percona and MySQL 8.0) `mysql` DB tables are created in the `mysql`
|
|
# tablespace, which is just ./mysql.ibd.
|
|
#
|
|
# Note that technically tablespace file locations could be quite arbitrary (see CREATE TABLESPACE and
|
|
# CREATE TABLE ... TABLESPACE syntax), but fetching that information (e.g. from INFORMATION_SCHEMA.FILES)
|
|
# generally requires the DB server to be up, which is not feasible. However, any tablespace file
|
|
# locations are expected to be under datadir, innodb_data_home_dir, innodb_directories system variables.
|
|
for file in "$datadir"/ibdata* "$datadir"/*/*.ibd "$datadir"/*.ibd; do
|
|
/usr/bin/innochecksum "$file" || {
|
|
rc="$?"
|
|
err "InnoDB tablespace file '$file' is corrupted."
|
|
}
|
|
done
|
|
[ "$rc" -eq 0 ] || report_disk_space_for_repair "$datadir"
|
|
[ "$rc" -ne 0 ] || mysql_action "start" || err "Could not start the database service."
|
|
} | result "corrupted"
|
|
exit 0
|
|
}
|
|
|
|
change_recovery_mode()
|
|
{
|
|
# This assumes that create_my_cnf_d() was called during installation, if needed.
|
|
local mode="$1"
|
|
local no_restart="$2"
|
|
|
|
local my_cnf_d="/etc/mysql/conf.d"
|
|
local config="$my_cnf_d/plesk-innodb-recovery.cnf"
|
|
|
|
if [ -n "$mode" ] && [ "$mode" -gt 0 ]; then
|
|
cat > "$config" <<-EOT
|
|
# Temporarily added by Plesk Repair Kit
|
|
[mysqld]
|
|
innodb_force_recovery = $mode
|
|
EOT
|
|
else
|
|
rm -f "$config"
|
|
fi
|
|
|
|
[ -n "$no_restart" ] || mysql_action "restart"
|
|
}
|
|
|
|
reinit_mysql_datadir()
|
|
{
|
|
local datadir="$1"
|
|
|
|
info "Re-initializing the database service data directory."
|
|
mysql_action "stop" || die "Could not stop the database service."
|
|
|
|
find "$datadir" -mindepth 1 -maxdepth 1 \
|
|
! \( -name debian-\*.flag -o -name 'lost+found' -o -name 'lost@002bfound' \) -print0 |
|
|
xargs -0 -r rm -rf
|
|
|
|
# remove innodb_force_recovery, as required by the mysql_install_db script
|
|
change_recovery_mode 0 --no-restart || die "Failed to de-configure the database recovery mode."
|
|
TMPDIR= log_if_failed bash /usr/bin/mysql_install_db \
|
|
--force --datadir="$datadir" --user=mysql --disable-log-bin ||
|
|
die "Failed to re-initialize '$datadir'."
|
|
|
|
# restart w/o innodb_force_recovery
|
|
info "Exiting InnoDB recovery mode."
|
|
change_recovery_mode || die "Failed to start the database server in normal mode."
|
|
}
|
|
|
|
database_client_invoke()
|
|
{
|
|
local use_root="$1"
|
|
shift
|
|
|
|
local MYSQL_BIN_D="/usr/bin"
|
|
local mysql_client="$MYSQL_BIN_D/mysql"
|
|
|
|
if [ -f "$MYSQL_BIN_D/mariadb" ]; then
|
|
mysql_client="$MYSQL_BIN_D/mariadb"
|
|
fi
|
|
|
|
if [ -n "$use_root" ]; then
|
|
if [ -s "/var/log/mysqld.log" ]; then
|
|
# MySQL generates and logs a temporary password (the last word on the line), forces to set a new one.
|
|
local marker='A temporary password is generated for root@localhost:'
|
|
local temp_passwd="`grep -F "$marker" "/var/log/mysqld.log" | tail -n1 | xargs -n1 | tail -n1`"
|
|
MYSQL_PWD="$temp_passwd" "$mysql_client" --connect-expired-password -uroot "$@" \
|
|
-e "ALTER USER 'root'@'localhost' IDENTIFIED BY '$temp_passwd';"
|
|
MYSQL_PWD="$temp_passwd" "$mysql_client" -uroot "$@"
|
|
else
|
|
"$mysql_client" -uroot "$@"
|
|
fi
|
|
else
|
|
MYSQL_PWD="`cat /etc/psa/.psa.shadow`" "$mysql_client" -uadmin "$@"
|
|
fi
|
|
}
|
|
|
|
db_convert_str()
|
|
{
|
|
local str="$1"
|
|
local src="$2"
|
|
local dst="$3"
|
|
|
|
# Note: this doesn't really account for 'lower_case_table_names', which may affect character case.
|
|
# https://mariadb.com/kb/en/identifier-case-sensitivity/
|
|
if [ -x "/usr/bin/mariadb-conv" ]; then
|
|
# Since MariaDB 10.5.1: https://mariadb.com/kb/en/mariadb-conv/
|
|
echo "$str" | /usr/bin/mariadb-conv -f "$src" -t "$dst" --delimiter="\r\n"
|
|
else
|
|
# A very dumb and incomplete fallback for other forks and versions.
|
|
if [ "$dst" = "filename" ]; then
|
|
[ -z "`echo "$str" | tr -d '[:alnum:]_'`" ]
|
|
elif [ "$src" = "filename" ]; then
|
|
[ "`echo "$str" | tr -d '@'`" = "$str" ]
|
|
fi || echo "Cannot correctly convert '$str' from '$src' to '$dst' on the current database service version." >&2
|
|
|
|
echo "$str"
|
|
fi
|
|
}
|
|
|
|
db_fname_to_utf8()
|
|
{
|
|
# On MariaDB same as: SELECT CONVERT(_filename $1 USING utf8);
|
|
# Implemented by 'my_charset_filename' and 'my_wc_mb_filename()' in MySQL sources.
|
|
# See also https://dev.mysql.com/doc/refman/8.4/en/identifier-mapping.html
|
|
# and https://mariadb.com/kb/en/identifier-to-file-name-mapping/
|
|
db_convert_str "$1" filename utf8
|
|
}
|
|
|
|
db_utf8_to_fname()
|
|
{
|
|
# Implemented by 'my_charset_filename' and 'my_mb_wc_filename()' in MySQL sources.
|
|
db_convert_str "$1" utf8 filename
|
|
}
|
|
|
|
db_fmt()
|
|
{
|
|
# Formats each argument DB filename for use in SQL
|
|
for db in "$@"; do
|
|
echo "\``db_fname_to_utf8 "$db" | sed -e 's|\`|\`\`|g'`\`"
|
|
done | xargs
|
|
}
|
|
|
|
file_age()
|
|
{
|
|
# Prints file age in human-readable form, such as '1d 3m 4s'
|
|
local path="$1"
|
|
[ -f "$path" ] || return 0
|
|
|
|
local age="$(($(date +%s) - $(date +%s -r "$path")))"
|
|
declare -A steps=(
|
|
[d]=$((24*3600))
|
|
[h]=3600
|
|
[m]=60
|
|
[s]=1
|
|
)
|
|
|
|
local result=() val=
|
|
for unit in d h m s; do
|
|
val=$((age / steps[$unit]))
|
|
age=$((age % steps[$unit]))
|
|
! [ "$val" -gt 0 ] || result+=("$val$unit")
|
|
done
|
|
|
|
echo "${result[*]}"
|
|
}
|
|
|
|
escape_for_regex()
|
|
{
|
|
# Escapes a string for use in sed or awk regex
|
|
echo -n "$1" | sed -e 's/[][\/$*.^]/\\&/g'
|
|
}
|
|
|
|
filter_db_from_sql_dump()
|
|
{
|
|
local db="`db_fmt "$1"`"
|
|
true escape_for_regex
|
|
awk "
|
|
BEGIN { p=1 }
|
|
/^$(escape_for_regex '-- Current Database:')/ { p=0 }
|
|
/^$(escape_for_regex "-- Current Database: $db")$/ { p=1 }
|
|
/^$(escape_for_regex '/*!40101 SET SQL_MODE=@OLD_SQL_MODE */;')$/ { p=1 }
|
|
/^$(escape_for_regex '/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;')$/ { p=1 }
|
|
/^$(escape_for_regex '/*!50112 SET @disable_bulk_load = ')/ { p=1 }
|
|
(p) { print }
|
|
"
|
|
}
|
|
|
|
is_db_in_sql_dump()
|
|
{
|
|
local db="`db_fmt "$1"`"
|
|
grep -Fxq -- "-- Current Database: $db"
|
|
}
|
|
|
|
is_plesk_db()
|
|
{
|
|
local db="$1"
|
|
# Some DBs may technically have different names (e.g. apsc), but we don't really expect this
|
|
# on modern Plesk versions and there's no reliable way to get them with unavailable MySQL.
|
|
local plesk_dbs=(
|
|
"psa"
|
|
"apsc"
|
|
"horde"
|
|
"mysql"
|
|
"phpmyadmin"
|
|
"roundcubemail"
|
|
"sitebuilder5"
|
|
)
|
|
printf "%s\n" "${plesk_dbs[@]}" | grep -Fxq "$db"
|
|
}
|
|
|
|
list_critical_dbs()
|
|
{
|
|
local critical_dbs=(
|
|
"psa"
|
|
"apsc"
|
|
"mysql"
|
|
)
|
|
printf "%s\n" "${critical_dbs[@]}"
|
|
}
|
|
|
|
is_critical_db()
|
|
{
|
|
local db="$1"
|
|
list_critical_dbs | grep -Fxq "$db"
|
|
}
|
|
|
|
list_databases_on_fs()
|
|
{
|
|
local datadir="$1"
|
|
# If this is not precise enough, we could check for directories with db.opt files inside
|
|
find "$datadir" -mindepth 1 -maxdepth 1 \
|
|
-type d ! \( -name 'lost+found' -o -name 'lost@002bfound' -o -name '*#*' \) \
|
|
-exec basename {} \;
|
|
}
|
|
|
|
should_restore_db()
|
|
{
|
|
local db="$1"
|
|
local backup="$2"
|
|
|
|
# Restore Plesk DBs unconditionally
|
|
! is_plesk_db "$db" || return 0
|
|
|
|
# Restore other DBs as long as their total size of non-compressed dumps doesn't exceed 625 Mb.
|
|
# This is an estimate of 0.75 percentile across all production servers (assuming 5x compression).
|
|
[ -n "$LIMIT_OF_RESTORED_USER_DB_SIZE" ] || LIMIT_OF_RESTORED_USER_DB_SIZE=$((625 * 1024 * 1024))
|
|
|
|
gunzip < "$backup" 2>/dev/null | is_db_in_sql_dump "$db" || return 1
|
|
|
|
local dump_size="`gunzip < "$backup" | filter_db_from_sql_dump "$db" | wc -c`"
|
|
if [ "$(( LIMIT_OF_RESTORED_USER_DB_SIZE - dump_size ))" -ge 0 ]; then
|
|
LIMIT_OF_RESTORED_USER_DB_SIZE="$(( LIMIT_OF_RESTORED_USER_DB_SIZE - dump_size ))"
|
|
return 0
|
|
fi
|
|
|
|
return 1
|
|
}
|
|
|
|
repair()
|
|
{
|
|
local recovery_modes=(${PLESK_REPAIR_INNODB_RECOVERY_MODES:-1 2 3 4 5 6})
|
|
local datadir="`get_mysql_datadir`"
|
|
local corrupted_ibdata=()
|
|
local corrupted_dbs=()
|
|
local all_dbs=()
|
|
local failed_critical_dbs=()
|
|
local missing_dbs=()
|
|
local backup_d=
|
|
local need_datadir_reinit=
|
|
declare -A backups=()
|
|
# Special $backups key for all databases - some invalid name for a database - more than 64 chars
|
|
local -r ALL="__ALL_DATABASES__________________________________________________"
|
|
|
|
! is_remote_db_feature_enabled || die "Cannot repair InnoDB corruption on a remote database."
|
|
|
|
# 0. check which databases are affected
|
|
info "(1/6) Checking what should be repaired."
|
|
|
|
do_check_for_corrupted_files()
|
|
{
|
|
info "Checking for corrupted InnoDB tablespace files."
|
|
mysql_action "stop" || die "Could not stop the database service."
|
|
|
|
corrupted_ibdata=()
|
|
for file in "$datadir"/ibdata*; do
|
|
/usr/bin/innochecksum "$file" >/dev/null 2>&1 ||
|
|
corrupted_ibdata+=("`basename "$file"`")
|
|
done
|
|
|
|
corrupted_dbs=()
|
|
for file in "$datadir"/*/*.ibd; do
|
|
/usr/bin/innochecksum "$file" >/dev/null 2>&1 ||
|
|
corrupted_dbs+=("$(basename "`dirname "$file"`")")
|
|
done
|
|
|
|
# Heuristic, mostly for ./mysql.ibd (separate tablespace for `mysql` DB).
|
|
for file in "$datadir"/*.ibd; do
|
|
/usr/bin/innochecksum "$file" >/dev/null 2>&1 ||
|
|
corrupted_dbs+=("$(basename "${file%.*}")")
|
|
done
|
|
|
|
corrupted_dbs=(`printf "%s\n" "${corrupted_dbs[@]}" | sort -u`)
|
|
|
|
# This check is mostly to foolproof the verification stage.
|
|
failed_critical_dbs=()
|
|
for db in `list_critical_dbs`; do
|
|
[ -d "$datadir/$db" ] || failed_critical_dbs+=("$db")
|
|
done
|
|
|
|
[ -z "${corrupted_ibdata[*]}" ] ||
|
|
warn "The following system tablespace files are corrupted: ${corrupted_ibdata[*]}"
|
|
[ -z "${corrupted_dbs[*]}" ] ||
|
|
warn "The following databases contain corrupted tablespace files: ${corrupted_dbs[*]}"
|
|
[ -z "${failed_critical_dbs[*]}" ] ||
|
|
warn "The following essential databases are missing: ${failed_critical_dbs[*]}"
|
|
|
|
[ -z "${corrupted_ibdata[*]}" -a -z "${corrupted_dbs[*]}" -a -z "${failed_critical_dbs[*]}" ]
|
|
}
|
|
|
|
mysql_disable_respawn
|
|
|
|
! do_check_for_corrupted_files || {
|
|
mysql_action "start" || die "Failed to start the database service after check for corruption."
|
|
success "There are no corrupted InnoDB tablespace files - nothing to repair."
|
|
}
|
|
|
|
# Check if data dir will need to be re-initialized. 'mysql' cannot be reliably repaired alone.
|
|
if [ -n "${corrupted_ibdata[*]}" ] || printf "%s\n" "${corrupted_dbs[@]}" | grep -Fxq "mysql"; then
|
|
need_datadir_reinit="yes"
|
|
fi
|
|
|
|
all_dbs=(`list_databases_on_fs "$datadir"`)
|
|
|
|
# 1. back up MySQL datadir
|
|
info "(2/6) Backing up the database service data directory."
|
|
|
|
backup_d="`create_backup_dir`" ||
|
|
die "Cannot create directory to store backups."
|
|
log_backup_dir_location "$backup_d"
|
|
|
|
cp -raT --reflink=auto "$datadir" "$backup_d/mysql" ||
|
|
die "Cannot backup '$datadir' to '$backup_d/mysql'. Ensure sufficient disk space is available."
|
|
|
|
info "(3/6) Backing up the affected databases."
|
|
|
|
do_backup_database()
|
|
{
|
|
local db="$1"
|
|
local mysqldump="$PRODUCT_ROOT_D/admin/sbin/mysqldump.sh"
|
|
if [ "$db" = "$ALL" ]; then
|
|
"$mysqldump" --dump-dir "$backup_d" --title "$ALL" --all-databases
|
|
else
|
|
"$mysqldump" --dump-dir "$backup_d" --title "$db" --databases "`db_fname_to_utf8 "$db"`"
|
|
fi
|
|
}
|
|
|
|
do_backup_databases()
|
|
{
|
|
local rc=0
|
|
local log_path="$PRODUCT_LOGS_D/mysqldump.log"
|
|
|
|
for db in "$@"; do
|
|
[ -z "${backups["$db"]}" ] || continue
|
|
backups["$db"]="`do_backup_database "$db"`" || {
|
|
backups["$db"]=
|
|
rc=1
|
|
[ "$db" = "$ALL" ] || warn "Failed to dump '$db' database. See $log_path for details."
|
|
}
|
|
done
|
|
|
|
return "$rc"
|
|
}
|
|
|
|
for mode in "${recovery_modes[@]}"; do
|
|
# 2. try innodb_force_recovery
|
|
info "Trying to backup databases in InnoDB recovery mode $mode."
|
|
change_recovery_mode "$mode" || {
|
|
warn "Failed to start the database server in InnoDB recovery mode $mode."
|
|
continue
|
|
}
|
|
|
|
# 3. dump databases (not tables)
|
|
if [ -n "$need_datadir_reinit" ]; then
|
|
do_backup_databases "$ALL" || {
|
|
warn "Failed to dump all databases. Will try to dump databases one by one."
|
|
do_backup_databases "${all_dbs[@]}" || continue
|
|
}
|
|
else
|
|
do_backup_databases "${corrupted_dbs[@]}" || continue
|
|
fi
|
|
|
|
break
|
|
done
|
|
|
|
[ -n "${backups["$ALL"]}" ] || backups["$ALL"]="`find_latest_preexisting_dump`"
|
|
[ -n "${backups["$ALL"]}" ] || warn "No pre-existing Plesk databases dump found."
|
|
|
|
info "Finished backup into '$backup_d'. Directory size: `du -shD "$backup_d" | cut -f1`."
|
|
|
|
info "(4/6) Removing the corrupted data."
|
|
|
|
if [ -n "$need_datadir_reinit" ]; then
|
|
# 4-5. re-initialize MySQL data dir, all DBs will effectively be dropped
|
|
reinit_mysql_datadir "$datadir"
|
|
else
|
|
# 4. drop corrupted databases
|
|
info "Dropping the corrupted databases."
|
|
for db in "${corrupted_dbs[@]}"; do
|
|
# In case db_fmt() wasn't able to correctly convert the DB name, it's better to skip the drop
|
|
# (which happens due to 'IF EXISTS') than to remove the files manually, since this will prevent
|
|
# the respective tables from being re-created (w/o re-initialization of the MySQL data dir).
|
|
log_if_failed database_client_invoke "$need_datadir_reinit" -e "DROP DATABASE IF EXISTS `db_fmt "$db"`;" || {
|
|
warn "Failed to drop the corrupted database '$db'. Will remove it from the filesystem."
|
|
rm -rf "$datadir/$db" || warn "Failed to remove '$datadir/$db'."
|
|
}
|
|
done
|
|
|
|
# 5. restart w/o innodb_force_recovery
|
|
info "Exiting InnoDB recovery mode."
|
|
change_recovery_mode || die "Failed to start the database server in normal mode."
|
|
fi
|
|
|
|
# 6. restore databases from available backups
|
|
info "(5/6) Restoring the removed databases from backups."
|
|
|
|
do_restore_backup()
|
|
{
|
|
local db="$1"
|
|
local backup="${2:-${backups["$db"]}}"
|
|
local use_root="$need_datadir_reinit"
|
|
|
|
[ -f "$backup" ] || return 1
|
|
|
|
if [ "$db" = "$ALL" ]; then
|
|
gunzip < "$backup" |
|
|
log_if_failed database_client_invoke "$use_root" || return 1
|
|
info "Databases were restored from '$backup' (backup age: `file_age "$backup"`)."
|
|
else
|
|
gunzip < "$backup" | filter_db_from_sql_dump "$db" |
|
|
log_if_failed database_client_invoke "$use_root" || return 1
|
|
info "Database '$db' (`db_fmt "$db"`) was restored from '$backup' (backup age: `file_age "$backup"`)."
|
|
fi
|
|
}
|
|
|
|
do_restore_databases()
|
|
{
|
|
local fallback_to_all="$1"
|
|
shift
|
|
|
|
failed_critical_dbs=()
|
|
for db in "$@"; do
|
|
if [ -f "${backups["$db"]}" ]; then
|
|
should_restore_db "$db" "${backups["$db"]}" || {
|
|
warn "Database '$db' (`db_fmt "$db"`) will not be restored from '${backups["$db"]}'" \
|
|
"to limit repair time."
|
|
continue
|
|
}
|
|
elif [ -n "$fallback_to_all" -a -f "${backups["$ALL"]}" ]; then
|
|
should_restore_db "$db" "${backups["$ALL"]}" || {
|
|
warn "Database '$db' (`db_fmt "$db"`) will not be restored from '${backups["$ALL"]}'" \
|
|
"(either not present or too big)."
|
|
continue
|
|
}
|
|
else
|
|
warn "Database '$db' (`db_fmt "$db"`) will not be restored as there is no backup available."
|
|
continue
|
|
fi
|
|
|
|
! do_restore_backup "$db" || continue
|
|
[ -z "$fallback_to_all" ] ||
|
|
! do_restore_backup "$db" "${backups["$ALL"]}" || continue
|
|
|
|
warn "Failed to restore the '$db' (`db_fmt "$db"`) database from" \
|
|
"'${backups["$db"]}'${fallback_to_all:+ or '${backups["$ALL"]}'}."
|
|
|
|
! is_critical_db "$db" || failed_critical_dbs+=("$db")
|
|
done
|
|
}
|
|
|
|
do_check_for_missing_databases()
|
|
{
|
|
missing_dbs=()
|
|
for db in "${all_dbs[@]}"; do
|
|
[ -d "$datadir/$db" ] || missing_dbs+=("$db")
|
|
done
|
|
}
|
|
|
|
do_post_reinit_mysql()
|
|
{
|
|
# Mostly to avoid errors in Docker. Don't ask me why.
|
|
mysql_action "stop"
|
|
|
|
# Mainly to remove test DB and extra service users, in case they were re-created.
|
|
"$PRODUCT_ROOT_D/admin/sbin/mysqlmng" --post-re-init ||
|
|
warn "Failed to re-configure the database server after re-initializing its data directory."
|
|
}
|
|
|
|
if [ -n "$need_datadir_reinit" ]; then
|
|
do_restore_backup "$ALL" ||
|
|
warn "Failed to restore databases from '${backups["$ALL"]}'." \
|
|
"Will try to restore from individual backups, if any."
|
|
|
|
do_check_for_missing_databases
|
|
do_restore_databases "" "${missing_dbs[@]}"
|
|
|
|
do_post_reinit_mysql
|
|
else
|
|
do_restore_databases --fallback-to-all "${corrupted_dbs[@]}"
|
|
fi
|
|
|
|
# 7. verify
|
|
info "(6/6) Checking if the repair was successful."
|
|
|
|
[ -z "${failed_critical_dbs[*]}" ] ||
|
|
die "Failed to restore some essential databases: ${failed_critical_dbs[*]}"
|
|
|
|
do_check_for_corrupted_files ||
|
|
die "Failed to fully repair InnoDB tablespace files."
|
|
|
|
mysql_action "start" || die "Failed to start the database service after repair."
|
|
|
|
do_check_for_missing_databases
|
|
[ -z "${missing_dbs[*]}" ] || {
|
|
warn "The following databases were not restored, you will need to restore them manually," \
|
|
"for example from daily or subscription backups: ${missing_dbs[*]}"
|
|
|
|
local missing_dbs_sql_str="`db_fmt "${missing_dbs[@]}"`"
|
|
if [ "$(echo "$missing_dbs_sql_str" | tr -d '`')" != "${missing_dbs[*]}" ]; then
|
|
warn "Here are the same databases, but formatted for SQL rather than filesystem:" \
|
|
"$missing_dbs_sql_str"
|
|
elif [ -n "$(echo "$missing_dbs_sql_str" | tr -d '[:alnum:]_ `')" ]; then
|
|
warn "Most likely this happened because it was impossible to automatically convert" \
|
|
"database names found on the filesystem (as listed above) into names formatted for SQL."
|
|
fi
|
|
}
|
|
|
|
success "InnoDB tablespace files corruption has been repaired." \
|
|
"Make sure the restored database data is up-to-date and complete."
|
|
|
|
unset do_check_for_corrupted_files
|
|
unset do_backup_database
|
|
unset do_backup_databases
|
|
unset do_restore_backup
|
|
unset do_restore_databases
|
|
unset do_check_for_missing_databases
|
|
unset do_post_reinit_mysql
|
|
}
|
|
|
|
case "$action" in
|
|
--check)
|
|
check
|
|
;;
|
|
--repair)
|
|
repair
|
|
;;
|
|
*)
|
|
usage
|
|
;;
|
|
esac
|
|
|
|
# vim:ft=sh
|