Compare commits

..

18 Commits

Author SHA1 Message Date
Asif Bacchus
010ab1dceb perf(RESTORE): exit if nothing to do 2021-02-09 06:49:33 -07:00
Asif Bacchus
f28ad5e6e7 refactor(RESTORE): add logging for mailcow start/stop 2021-02-09 06:45:30 -07:00
Asif Bacchus
06df565348 style(RESTORE): add space before append redirects 2021-02-09 06:42:27 -07:00
Asif Bacchus
ea0d98734a fix(RESTORE): fix accidental log file overwrite vs append 2021-02-09 06:39:38 -07:00
Asif Bacchus
b07ad2c0ee perf(RESTORE): use docker-compose stop timeout 2021-02-09 06:01:04 -07:00
Asif Bacchus
84605567cd feature(RESTORE): restart mailcow after completion 2021-02-09 05:59:37 -07:00
Asif Bacchus
8c6019f7da feature(RESTORE): process secondary volume restoration 2021-02-09 05:48:17 -07:00
Asif Bacchus
363c970f2f refactor(RESTORE): create and use restore function
- single function to handle non-SQL restore operations
- replace previous mail and crypt restore with new function
2021-02-09 05:41:56 -07:00
Asif Bacchus
cc30581382 feature(RESTORE): restore mail and encryption key 2021-02-09 05:00:08 -07:00
Asif Bacchus
c9b85d084e feature(RESTORE): stop docker containers and confirm 2021-02-08 11:30:39 -07:00
Asif Bacchus
bf18c80f94 refactor(RESTORE): execute docker-compose silently 2021-02-08 10:59:47 -07:00
Asif Bacchus
764f35a695 docs(RESTORE): update error message list 2021-02-08 10:53:14 -07:00
Asif Bacchus
ba212f738f feature(RESTORE): clearer more accurate exit report messages
- change exit messages based on errors, warnings, etc.
- separate error codes for exit with non-crit err and/or warnings
2021-02-08 10:52:17 -07:00
Asif Bacchus
59cc9a5d66 fix(RESTORE): increment errorCount on SQL import error 2021-02-08 10:39:35 -07:00
Asif Bacchus
7b3ff7f6ae fix(RESTORE): strip trailing slash from backup-location 2021-02-08 10:31:03 -07:00
Asif Bacchus
78e1ed2539 perf(RESTORE): remove duplicate backup-location existence check 2021-02-08 10:28:21 -07:00
Asif Bacchus
af4fcf3cf3 fix(RESTORE): exit on invalid backup location 2021-02-08 10:25:24 -07:00
Asif Bacchus
561437d667 fix(RESTORE): fix printf err 2021-02-08 10:24:19 -07:00

View File

@ -47,6 +47,27 @@ exitError() {
exit "$1" exit "$1"
} }
doRestore() {
sourceFiles=$(find "${backupLocation}" -iname "${1}" -type d)
if [ -n "$sourceFiles" ]; then
if [ "$verbose" -eq 1 ]; then
if (! (cd "$sourceFiles/_data" && tar -cf - .) | (cd "${2}" && tar xvf -) >> "$logfile" ); then
return 1
else
return 0
fi
else
if (! (cd "$sourceFiles/_data" && tar -cf - .) | (cd "${3}" && tar xvf -) ); then
return 1
else
return 0
fi
fi
else
return 2
fi
}
scriptHelp() { scriptHelp() {
textNewline textNewline
printf "%sUsage: %s [parameters]%s\n\n" "$bold" "$scriptName" "$norm" printf "%sUsage: %s [parameters]%s\n\n" "$bold" "$scriptName" "$norm"
@ -84,7 +105,6 @@ textNewline() {
trapExit() { trapExit() {
printf "%s[%s] -- [ERROR] 99: Caught signal --%s\n" "$err" "$(stamp)" "$norm" >> "$logfile" printf "%s[%s] -- [ERROR] 99: Caught signal --%s\n" "$err" "$(stamp)" "$norm" >> "$logfile"
cleanup
printf "%s[%s] --- %s execution terminated via signal ---\n%s" "$err" "$(stamp)" "$scriptName" "$norm" >> "$logfile" printf "%s[%s] --- %s execution terminated via signal ---\n%s" "$err" "$(stamp)" "$scriptName" "$norm" >> "$logfile"
exit 99 exit 99
} }
@ -202,7 +222,7 @@ while [ $# -gt 0 ]; do
-b|--backup-location) -b|--backup-location)
if [ -n "$2" ]; then if [ -n "$2" ]; then
if [ -d "$2" ] && [ -n "$( ls -A "$2" )" ]; then if [ -d "$2" ] && [ -n "$( ls -A "$2" )" ]; then
backupLocation="$2" backupLocation="${2%/}"
shift shift
else else
consoleError '1' "$1: cannot find specified backup location directory or it is empty." consoleError '1' "$1: cannot find specified backup location directory or it is empty."
@ -227,7 +247,7 @@ while [ $# -gt 0 ]; do
restoreRspamd=0 restoreRspamd=0
;; ;;
*) *)
printf "\n%Unknown option: %s\n" "$err" "$1" printf "\n%sUnknown option: %s\n" "$err" "$1"
printf "Use '--help' for valid options.%s\n\n" "$norm" printf "Use '--help' for valid options.%s\n\n" "$norm"
exit 1 exit 1
;; ;;
@ -236,9 +256,13 @@ while [ $# -gt 0 ]; do
done done
### pre-flight checks ### pre-flight checks
# ensure there's something to do
if [ "$restoreMail" -eq 0 ] && [ "$restoreSQL" -eq 0 ] && [ "$restorePostfix" -eq 0 ] && [ "$restoreRedis" -eq 0 ] && [ "$restoreRedis" -eq 0 ]; then
printf "\n%sAll restore operations skipped -- nothing to do!%s\n\n" "$yellow" "$norm"
exit 0
fi
# set path so checks are valid for this script environment # set path so checks are valid for this script environment
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
# docker installed? # docker installed?
if ! command -v docker >/dev/null; then if ! command -v docker >/dev/null; then
consoleError '3' 'docker does not seem to be installed!' consoleError '3' 'docker does not seem to be installed!'
@ -251,8 +275,12 @@ fi
if [ ! -f "$mcDockerCompose" ]; then if [ ! -f "$mcDockerCompose" ]; then
consoleError '1' "docker-compose configuration ($mcDockerCompose) cannot be found." consoleError '1' "docker-compose configuration ($mcDockerCompose) cannot be found."
fi fi
# forgot to set backup location?
if [ -z "$backupLocation" ]; then
consoleError '1' "'--backup-location' cannot be unspecified or null/empty."
fi
# change to mailcow directory so commands execute properly # change to mailcow directory so commands execute properly
\cd ${mcConfig%/*} || consoleError '4' 'Cannot change to mailcow directory as determined from mailcow.conf location.' \cd "${mcConfig%/*}" || consoleError '4' 'Cannot change to mailcow directory as determined from mailcow.conf location.'
### read mailcow.conf and import vars ### read mailcow.conf and import vars
# shellcheck source=./mailcow.conf.shellcheck # shellcheck source=./mailcow.conf.shellcheck
@ -314,7 +342,7 @@ if [ "$restoreSQL" -eq 1 ]; then
if [ -n "$sqlBackup" ]; then if [ -n "$sqlBackup" ]; then
# start mysql container if not already running # start mysql container if not already running
if ! docker container inspect -f '{{ .State.Running }}' ${COMPOSE_PROJECT_NAME}_mysql-mailcow_1 > /dev/null 2>&1; then if ! docker container inspect -f '{{ .State.Running }}' ${COMPOSE_PROJECT_NAME}_mysql-mailcow_1 > /dev/null 2>&1; then
docker-compose up -d mysql-mailcow docker-compose up -d mysql-mailcow > /dev/null 2>&1
if docker container inspect -f '{{ .State.Running }}' ${COMPOSE_PROJECT_NAME}_mysql-mailcow_1 > /dev/null 2>&1; then if docker container inspect -f '{{ .State.Running }}' ${COMPOSE_PROJECT_NAME}_mysql-mailcow_1 > /dev/null 2>&1; then
sqlRunning=1 sqlRunning=1
else else
@ -338,27 +366,223 @@ if [ "$restoreSQL" -eq 1 ]; then
else else
writeLog 'done' 'error' writeLog 'done' 'error'
writeLog 'error' '13' "Something went wrong while trying to restore SQL database. Perhaps try again?" writeLog 'error' '13' "Something went wrong while trying to restore SQL database. Perhaps try again?"
errorCount=$((errorCount+1))
fi fi
fi fi
fi fi
#TODO: stop containers ### stop containers (necessary for all restore operations except SQL)
#TODO: copy backups to correct docker volumes writeLog 'task' "Stopping mailcow"
#TODO: restart docker containers if ! docker-compose down --timeout "${dockerStopTimeout}" > /dev/null 2>&1; then
#TODO: optionally reindex dovecot (parameter) writeLog 'done' 'error'
writeLog 'error' '20' "Unable to bring mailcow containers down -- cannot reliably restore. Aborting."
exitError 20
fi
if [ "$( docker ps --filter "name=${COMPOSE_PROJECT_NAME}" -q | wc -l )" -gt 0 ]; then
writeLog 'done' 'error'
writeLog 'error' '20' "Unable to bring mailcow containers down -- cannot reliably restore. Aborting."
exitError 20
fi
writeLog 'done'
### restore mail and encryption key
if [ "$restoreMail" -eq 1 ]; then
if [ "$verbose" -eq 1 ]; then
writeLog 'info' "Restoring email"
else
writeLog 'task' "Restoring email"
fi
# restore email messages
doRestore "${COMPOSE_PROJECT_NAME}_vmail-vol-1" "$dockerVolumeMail"; ec="$?"
case "$ec" in
0)
if [ "$verbose" -eq 1 ]; then
writeLog 'success' "Email messages restored"
else
writeLog 'done'
fi
;;
1)
if [ "$verbose" -eq 1 ]; then
writeLog 'error' '52' "There was an error restoring one or more email messages."
else
writeLog 'done' 'error'
writeLog 'error' '52' "There was an error restoring one or more email messages."
fi
;;
2)
if [ "$verbose" -eq 1 ]; then
writeLog 'error' '51' "Cannot locate email message backups!"
else
writeLog 'done' 'error'
writeLog 'error' '51' "Cannot locate email message backups!"
fi
;;
esac
# restore encryption key
doRestore "${COMPOSE_PROJECT_NAME}_crypt-vol-1" "$dockerVolumeCrypt"; ec="$?"
case "$ec" in
0)
if [ "$verbose" -eq 1 ]; then
writeLog 'success' "Encryption key restored"
else
writeLog 'done'
fi
;;
1)
if [ "$verbose" -eq 1 ]; then
writeLog 'error' '52' "There was an error restoring the encryption key! Any restored messages are likely *not* readable!"
else
writeLog 'done' 'error'
writeLog 'error' '52' "There was an error restoring the encryption key! Any restored messages are likely *not* readable!"
fi
;;
2)
if [ "$verbose" -eq 1 ]; then
writeLog 'error' '51' "Cannot locate encryption key backup!"
else
writeLog 'done' 'error'
writeLog 'error' '51' "Cannot locate encryption key backup!"
fi
;;
esac
fi
### restore postfix
if [ "$restorePostfix" -eq 1 ]; then
if [ "$verbose" -eq 1 ]; then
writeLog 'info' "Restoring postfix files"
else
writeLog 'task' "Restoring postfix files"
fi
# restore email messages
doRestore "${COMPOSE_PROJECT_NAME}_postfix-vol-1" "$dockerVolumePostfix"; ec="$?"
case "$ec" in
0)
if [ "$verbose" -eq 1 ]; then
writeLog 'success' "Postfix files restored"
else
writeLog 'done'
fi
;;
1)
if [ "$verbose" -eq 1 ]; then
writeLog 'error' '52' "There was an error restoring one or more postfix files."
else
writeLog 'done' 'error'
writeLog 'error' '52' "There was an error restoring one or more postfix files."
fi
;;
2)
if [ "$verbose" -eq 1 ]; then
writeLog 'error' '51' "Cannot locate postfix backups!"
else
writeLog 'done' 'error'
writeLog 'error' '51' "Cannot locate postfix backups!"
fi
;;
esac
fi
### restore rspamd
if [ "$restoreRspamd" -eq 1 ]; then
if [ "$verbose" -eq 1 ]; then
writeLog 'info' "Restoring Rspamd files"
else
writeLog 'task' "Restoring Rspamd files"
fi
# restore email messages
doRestore "${COMPOSE_PROJECT_NAME}_rspamd-vol-1" "$dockerVolumeRspamd"; ec="$?"
case "$ec" in
0)
if [ "$verbose" -eq 1 ]; then
writeLog 'success' "Rspamd files restored"
else
writeLog 'done'
fi
;;
1)
if [ "$verbose" -eq 1 ]; then
writeLog 'error' '52' "There was an error restoring one or more Rspamd files."
else
writeLog 'done' 'error'
writeLog 'error' '52' "There was an error restoring one or more Rspamd files."
fi
;;
2)
if [ "$verbose" -eq 1 ]; then
writeLog 'error' '51' "Cannot locate Rspamd backups!"
else
writeLog 'done' 'error'
writeLog 'error' '51' "Cannot locate Rspamd backups!"
fi
;;
esac
fi
### restore redis
if [ "$restoreRedis" -eq 1 ]; then
if [ "$verbose" -eq 1 ]; then
writeLog 'info' "Restoring redis database"
else
writeLog 'task' "Restoring redis database"
fi
# restore email messages
doRestore "${COMPOSE_PROJECT_NAME}_redis-vol-1" "$dockerVolumeRedis"; ec="$?"
case "$ec" in
0)
if [ "$verbose" -eq 1 ]; then
writeLog 'success' "Redis database restored"
else
writeLog 'done'
fi
;;
1)
if [ "$verbose" -eq 1 ]; then
writeLog 'error' '52' "There was an error restoring the redis database. This is usually *not* a serious issue."
else
writeLog 'done' 'error'
writeLog 'error' '52' "There was an error restoring the redis database. This is usually *not* a serious issue."
fi
;;
2)
if [ "$verbose" -eq 1 ]; then
writeLog 'error' '51' "Cannot locate redis database backups!"
else
writeLog 'done' 'error'
writeLog 'error' '51' "Cannot locate redis database backups!"
fi
;;
esac
fi
### restart mailcow
writeLog 'task' "Starting mailcow"
if ! docker-compose up -d > /dev/null 2>&1; then
writeLog 'done' 'warn'
writeLog 'warn' '21' "Unable to automatically start mailcow containers. Please attempt a manual start and note any errors."
warnCount=$((warnCount+1))
fi
writeLog 'done'
### exit gracefully ### exit gracefully
if [ "$errorCount" -gt 0 ]; then
# note non-terminating errors
printf "%s[%s] --- %s execution completed with %s error(s) ---\n%s" "$err" "$(stamp)" "$scriptName" "$errorCount" "$norm" >> "$logfile"
exit 98
elif [ "$warnCount" -gt 0 ]; then
printf "%s[%s] --- %s execution completed with %s warning(s) ---\n%s" "$yellow" "$(stamp)" "$scriptName" "$warnCount" "$norm" >> "$logfile"
exit 97
else
writeLog 'success' "All processes completed" writeLog 'success' "All processes completed"
printf "%s[%s] --- %s execution completed ---\n%s" "$magenta" "$(stamp)" "$scriptName" "$norm" >> "$logfile" printf "%s[%s] --- %s execution completed ---\n%s" "$magenta" "$(stamp)" "$scriptName" "$norm" >> "$logfile"
# note non-terminating errors
if [ "$errorCount" -gt 0 ]; then
printf "%s%s errors encountered!%s\n" "$err" "$errorCount" "$norm" >>"$logfile"
fi
# note warnings
if [ "$warnCount" -gt 0 ]; then
printf "%s%s warnings issued!%s\n" "$yellow" "$warnCount" "$norm" >>"$logfile"
fi
exit 0 exit 0
fi
### error codes: ### error codes:
# 1: parameter error # 1: parameter error
@ -370,9 +594,14 @@ exit 0
# 11: cannot locate SQL dump in backup directory # 11: cannot locate SQL dump in backup directory
# 12: cannot start mysql-mailcow container # 12: cannot start mysql-mailcow container
# 13: restoring SQL dump was unsuccessful # 13: restoring SQL dump was unsuccessful
# 2x: Docker/Docker-Compose errors
# 20: cannot bring docker container(s) down successfully
# 21: cannot bring docker container(s) up successfully
# 5x: File restore errors
# 51: cannot locate source files in backup directory
# 52: error restoring one or more files
# 97: script completed with 1 or more warnings
# 98: script completed with 1 or more non-terminating errors
# 99: TERM signal trapped # 99: TERM signal trapped
# 100: could not change to mailcow-dockerized directory
# 101: could not stop container(s)
# 102: could not start container(s)
#EOF #EOF