26 Commits

Author SHA1 Message Date
Asif Bacchus 2747b10040 docs(RESTORE): remove misleading copied comments 2021-02-09 14:59:59 -07:00
Asif Bacchus 876f8eee9e fix(RESTORE): fix restore path when non-verbose
- non-verbose tar extraction was set to null path
2021-02-09 14:58:50 -07:00
Asif Bacchus 3a3b02685b make restore script executable 2021-02-09 08:30:06 -07:00
Asif Bacchus 85e34c73a3 feature(BACKUP): add compression option
- parameter to specify compression used by borg
- as requested by shaneonabike (via github)

Closes github 5
2021-02-09 08:22:48 -07:00
Asif Bacchus 9e592633b9 fix(BACKUP): bypass possible cd aliases 2021-02-09 08:03:23 -07:00
Asif Bacchus e127616800 fix(BACKUP): fix missing parameter shift
- docker timeout parameters
2021-02-09 08:02:34 -07:00
Asif Bacchus 053e45affc docs(RESTORE): add inline help 2021-02-09 07:46:42 -07:00
Asif Bacchus fb58c06489 fix(RESTORE): fix output to console when non-verbose 2021-02-09 07:13:12 -07:00
Asif Bacchus 010ab1dceb perf(RESTORE): exit if nothing to do 2021-02-09 06:49:33 -07:00
Asif Bacchus f28ad5e6e7 refactor(RESTORE): add logging for mailcow start/stop 2021-02-09 06:45:30 -07:00
Asif Bacchus 06df565348 style(RESTORE): add space before append redirects 2021-02-09 06:42:27 -07:00
Asif Bacchus ea0d98734a fix(RESTORE): fix accidental log file overwrite vs append 2021-02-09 06:39:38 -07:00
Asif Bacchus b07ad2c0ee perf(RESTORE): use docker-compose stop timeout 2021-02-09 06:01:04 -07:00
Asif Bacchus 84605567cd feature(RESTORE): restart mailcow after completion 2021-02-09 05:59:37 -07:00
Asif Bacchus 8c6019f7da feature(RESTORE): process secondary volume restoration 2021-02-09 05:48:17 -07:00
Asif Bacchus 363c970f2f refactor(RESTORE): create and use restore function
- single function to handle non-SQL restore operations
- replace previous mail and crypt restore with new function
2021-02-09 05:41:56 -07:00
Asif Bacchus cc30581382 feature(RESTORE): restore mail and encryption key 2021-02-09 05:00:08 -07:00
Asif Bacchus c9b85d084e feature(RESTORE): stop docker containers and confirm 2021-02-08 11:30:39 -07:00
Asif Bacchus bf18c80f94 refactor(RESTORE): execute docker-compose silently 2021-02-08 10:59:47 -07:00
Asif Bacchus 764f35a695 docs(RESTORE): update error message list 2021-02-08 10:53:14 -07:00
Asif Bacchus ba212f738f feature(RESTORE): clearer more accurate exit report messages
- change exit messages based on errors, warnings, etc.
- separate error codes for exit with non-crit err and/or warnings
2021-02-08 10:52:17 -07:00
Asif Bacchus 59cc9a5d66 fix(RESTORE): increment errorCount on SQL import error 2021-02-08 10:39:35 -07:00
Asif Bacchus 7b3ff7f6ae fix(RESTORE): strip trailing slash from backup-location 2021-02-08 10:31:03 -07:00
Asif Bacchus 78e1ed2539 perf(RESTORE): remove duplicate backup-location existence check 2021-02-08 10:28:21 -07:00
Asif Bacchus af4fcf3cf3 fix(RESTORE): exit on invalid backup location 2021-02-08 10:25:24 -07:00
Asif Bacchus 561437d667 fix(RESTORE): fix printf err 2021-02-08 10:24:19 -07:00
2 changed files with 324 additions and 38 deletions
+20 -2
View File
@@ -164,6 +164,10 @@ scriptHelp() {
textblock "Path to write log file"
defaultsTextblock "(scriptPath/scriptName.log)"
newline
switchTextblock "--compression"
textblock "Compression algorithm(s) that borg should use. Please run 'borg help compression' for details."
defaultsTextblock "(not specified, use borg default of lz4)"
newline
switchTextblock "[SWITCH] -v | --verbose"
textblock "Log borg output with increased verbosity (list all files). Careful! Your log file can get very large very quickly!"
defaultsTextblock "(normal output, option is OFF)"
@@ -300,6 +304,15 @@ while [ $# -gt 0 ]; do
badParam empty "$@"
fi
;;
--compression)
# set borg archive compression
if [ -n "$2" ]; then
borgCompression="$2"
shift
else
badParam empty "$@"
fi
;;
-v|--verbose)
# set verbose logging from borg
borgCreateParams='--list --stats'
@@ -367,6 +380,7 @@ while [ $# -gt 0 ]; do
badParam empty "$@"
else
dockerStartTimeout="$2"
shift
fi
;;
-t2|--timeout-stop)
@@ -374,6 +388,7 @@ while [ $# -gt 0 ]; do
badParam empty "$@"
else
dockerStopTimeout="$2"
shift
fi
;;
*)
@@ -417,7 +432,10 @@ fi
if [ ! -f "$mcDockerCompose" ]; then
badParam dne "(--docker-compose)" "$mcDockerCompose"
fi
# set compression level if specified
if [ -n "$borgCompression" ]; then
borgCreateParams="${borgCreateParams} --compression ${borgCompression}"
fi
### read mailcow.conf and set vars as needed
# shellcheck source=./mailcow.conf.shellcheck
@@ -615,7 +633,7 @@ if [ "$use503" -eq 1 ]; then
fi
### change to mailcow directory so docker commands execute properly
cd "${mcConfig%/*}" || exitError 100 'Could not change to mailcow directory.'
\cd "${mcConfig%/*}" || exitError 100 'Could not change to mailcow directory.'
### stop postfix and dovecot mail containers to prevent mailflow during backup
doDocker stop postfix
Regular → Executable
+304 -36
View File
@@ -43,15 +43,78 @@ consoleError() {
}
exitError() {
printf "%s[%s] --- %s execution completed with error ---\n%s" "$err" "$(stamp)" "$scriptName" "$norm" >>"$logfile"
printf "%s[%s] --- %s execution completed with error ---\n%s" "$err" "$(stamp)" "$scriptName" "$norm" >> "$logfile"
exit "$1"
}
doRestore() {
sourceFiles=$(find "${backupLocation}" -iname "${1}" -type d)
if [ -n "$sourceFiles" ]; then
if [ "$verbose" -eq 1 ]; then
if (! (cd "$sourceFiles/_data" && tar -cf - .) | (cd "${2}" && tar xvf -) >> "$logfile" ); then
return 1
else
return 0
fi
else
if (! (cd "$sourceFiles/_data" && tar -cf - .) | (cd "${2}" && tar xvf -) > /dev/null 2>&1 ); then
return 1
else
return 0
fi
fi
else
return 2
fi
}
scriptHelp() {
textNewline
printf "%sUsage: %s [parameters]%s\n\n" "$bold" "$scriptName" "$norm"
textblock "The only required parameter is -b | --backup-location."
textblock "If a parameter is not supplied, its default value will be used."
textblock "Switch parameters will only be activated if specified."
textblockHeader "script parameters"
textblockParam "-b | --backup-location"
textblock "Directory containing extracted backup files from borg repo. REQUIRED."
textNewline
textblock "If a parameter is not supplied, its default value will be used. Switch parameters will remain DEactivated if NOT specified."
textblockParam "-l | --log" "scriptPath/scriptName.log"
textblock "Path to write log file. Best efforts will be made to create any specified paths."
textNewline
textblockParam "-v | --verbose" "false"
textblock "Enable verbose logging. This will list EVERY restored file possibly making your log file quite large! [SWITCH]"
textNewline
textblockParam "--skip-mail" "false"
textblock "Skip restoring mail and encryption key. [SWITCH]"
textNewline
textblockParam "--skip-sql" "false"
textblock "Skip restoring mailcow settings database. [SWITCH]"
textNewline
textblockParam "--skip-postfix" "false"
textblock "Skip restoring postfix settings. [SWITCH]"
textNewline
textblockParam "--skip-rspamd" "false"
textblock "Skip restoring Rspamd settings/configuration/history. [SWITCH]"
textNewline
textblockParam "--skip-redis" "false"
textblock "Skip restoring redis database. [SWITCH]"
textNewline
textblockParam "-? | -h | --help"
textblock "Display this help screen."
textblockHeader "mailcow parameters"
textblockParam "-d | --docker-compose" "/opt/mailcow-dockerized/docker-compose.yml"
textblock "FULL path to mailcow's 'docker-compose.yml' file."
textNewline
textblockParam "-m | --mailcow-config" "/opt/mailcow-dockerized/mailcow.conf"
textblock "FULL path to mailcow configuration file ('mailcow.conf'). The path of this file is also used to determine your mailcow directory."
textblockHeader "docker parameters"
textblockParam "-t1 | --timeout-start" "180"
textblock "Seconds to wait for docker containers to start."
textNewline
textblockParam "-t2 | --timeout-stop" "120"
textblock "Seconds to wait for docker containers to stop."
textNewline
textblock "More details and examples of script usage can be found in the repo wiki at ${yellow}https://git.asifbacchus.app/asif/MailcowBackup/wiki${norm}"
textNewline
exit 0
}
@@ -65,7 +128,7 @@ textblock() {
}
textblockHeader() {
printf "\n%s%s***%s***%s\n" "$bold" "$magenta" "$1" "$norm"
printf "\n%s%s*** %s ***%s\n\n" "$bold" "$magenta" "$1" "$norm"
}
textblockParam() {
@@ -83,31 +146,30 @@ textNewline() {
}
trapExit() {
printf "%s[%s] -- [ERROR] 99: Caught signal --%s\n" "$err" "$(stamp)" "$norm" >>"$logfile"
cleanup
printf "%s[%s] --- %s execution terminated via signal ---\n%s" "$err" "$(stamp)" "$scriptName" "$norm" >>"$logfile"
printf "%s[%s] -- [ERROR] 99: Caught signal --%s\n" "$err" "$(stamp)" "$norm" >> "$logfile"
printf "%s[%s] --- %s execution terminated via signal ---\n%s" "$err" "$(stamp)" "$scriptName" "$norm" >> "$logfile"
exit 99
}
writeLog() {
if [ "$1" = "task" ]; then
printf "%s[%s] -- [INFO] %s... " "$info" "$(stamp)" "$2" >>"$logfile"
printf "%s[%s] -- [INFO] %s... " "$info" "$(stamp)" "$2" >> "$logfile"
elif [ "$1" = "done" ]; then
if [ -z "$2" ]; then
printf "%sdone%s --\n%s" "$ok" "$info" "$norm" >>"$logfile"
printf "%sdone%s --\n%s" "$ok" "$info" "$norm" >> "$logfile"
elif [ "$2" = "error" ]; then
printf "%sERROR%s --\n%s" "$err" "$info" "$norm" >>"$logfile"
printf "%sERROR%s --\n%s" "$err" "$info" "$norm" >> "$logfile"
elif [ "$2" = "warn" ]; then
printf "%swarning%s --\n%s" "$yellow" "$info" "$norm" >>"$logfile"
printf "%swarning%s --\n%s" "$yellow" "$info" "$norm" >> "$logfile"
fi
elif [ "$1" = "error" ]; then
printf "%s[%s] -- [ERROR] %s: %s --\n%s" "$err" "$(stamp)" "$2" "$3" "$norm" >>"$logfile"
printf "%s[%s] -- [ERROR] %s: %s --\n%s" "$err" "$(stamp)" "$2" "$3" "$norm" >> "$logfile"
elif [ "$1" = "warn" ]; then
printf "%s[%s] -- [WARNING] %s --\n%s" "$yellow" "$(stamp)" "$2" "$norm" >>"$logfile"
printf "%s[%s] -- [WARNING] %s --\n%s" "$yellow" "$(stamp)" "$2" "$norm" >> "$logfile"
elif [ "$1" = "info" ]; then
printf "%s[%s] -- [INFO] %s --\n%s" "$info" "$(stamp)" "$2" "$norm" >>"$logfile"
printf "%s[%s] -- [INFO] %s --\n%s" "$info" "$(stamp)" "$2" "$norm" >> "$logfile"
elif [ "$1" = "success" ]; then
printf "%s[%s] -- [SUCCESS] %s --\n%s" "$ok" "$(stamp)" "$2" "$norm" >>"$logfile"
printf "%s[%s] -- [SUCCESS] %s --\n%s" "$ok" "$(stamp)" "$2" "$norm" >> "$logfile"
fi
}
@@ -202,7 +264,7 @@ while [ $# -gt 0 ]; do
-b|--backup-location)
if [ -n "$2" ]; then
if [ -d "$2" ] && [ -n "$( ls -A "$2" )" ]; then
backupLocation="$2"
backupLocation="${2%/}"
shift
else
consoleError '1' "$1: cannot find specified backup location directory or it is empty."
@@ -227,7 +289,7 @@ while [ $# -gt 0 ]; do
restoreRspamd=0
;;
*)
printf "\n%Unknown option: %s\n" "$err" "$1"
printf "\n%sUnknown option: %s\n" "$err" "$1"
printf "Use '--help' for valid options.%s\n\n" "$norm"
exit 1
;;
@@ -236,9 +298,13 @@ while [ $# -gt 0 ]; do
done
### pre-flight checks
# ensure there's something to do
if [ "$restoreMail" -eq 0 ] && [ "$restoreSQL" -eq 0 ] && [ "$restorePostfix" -eq 0 ] && [ "$restoreRedis" -eq 0 ] && [ "$restoreRedis" -eq 0 ]; then
printf "\n%sAll restore operations skipped -- nothing to do!%s\n\n" "$yellow" "$norm"
exit 0
fi
# set path so checks are valid for this script environment
export PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
# docker installed?
if ! command -v docker >/dev/null; then
consoleError '3' 'docker does not seem to be installed!'
@@ -251,8 +317,12 @@ fi
if [ ! -f "$mcDockerCompose" ]; then
consoleError '1' "docker-compose configuration ($mcDockerCompose) cannot be found."
fi
# forgot to set backup location?
if [ -z "$backupLocation" ]; then
consoleError '1' "'--backup-location' cannot be unspecified or null/empty."
fi
# change to mailcow directory so commands execute properly
\cd ${mcConfig%/*} || consoleError '4' 'Cannot change to mailcow directory as determined from mailcow.conf location.'
\cd "${mcConfig%/*}" || consoleError '4' 'Cannot change to mailcow directory as determined from mailcow.conf location.'
### read mailcow.conf and import vars
# shellcheck source=./mailcow.conf.shellcheck
@@ -283,7 +353,7 @@ else
fi
fi
# write initial log entries
if ! printf "%s[%s] --- Start %s execution ---\n%s" "$magenta" "$(stamp)" "$scriptName" "$norm" 2>/dev/null >>"$logfile"; then
if ! printf "%s[%s] --- Start %s execution ---\n%s" "$magenta" "$(stamp)" "$scriptName" "$norm" 2>/dev/null >> "$logfile"; then
consoleError '1' "Unable to write to log file ($logfile)"
fi
writeLog 'info' "Log located at $logfile"
@@ -314,7 +384,7 @@ if [ "$restoreSQL" -eq 1 ]; then
if [ -n "$sqlBackup" ]; then
# start mysql container if not already running
if ! docker container inspect -f '{{ .State.Running }}' ${COMPOSE_PROJECT_NAME}_mysql-mailcow_1 > /dev/null 2>&1; then
docker-compose up -d mysql-mailcow
docker-compose up -d mysql-mailcow > /dev/null 2>&1
if docker container inspect -f '{{ .State.Running }}' ${COMPOSE_PROJECT_NAME}_mysql-mailcow_1 > /dev/null 2>&1; then
sqlRunning=1
else
@@ -338,27 +408,220 @@ if [ "$restoreSQL" -eq 1 ]; then
else
writeLog 'done' 'error'
writeLog 'error' '13' "Something went wrong while trying to restore SQL database. Perhaps try again?"
errorCount=$((errorCount+1))
fi
fi
fi
#TODO: stop containers
#TODO: copy backups to correct docker volumes
#TODO: restart docker containers
#TODO: optionally reindex dovecot (parameter)
### stop containers (necessary for all restore operations except SQL)
writeLog 'task' "Stopping mailcow"
if ! docker-compose down --timeout "${dockerStopTimeout}" > /dev/null 2>&1; then
writeLog 'done' 'error'
writeLog 'error' '20' "Unable to bring mailcow containers down -- cannot reliably restore. Aborting."
exitError 20
fi
if [ "$( docker ps --filter "name=${COMPOSE_PROJECT_NAME}" -q | wc -l )" -gt 0 ]; then
writeLog 'done' 'error'
writeLog 'error' '20' "Unable to bring mailcow containers down -- cannot reliably restore. Aborting."
exitError 20
fi
writeLog 'done'
### restore mail and encryption key
if [ "$restoreMail" -eq 1 ]; then
if [ "$verbose" -eq 1 ]; then
writeLog 'info' "Restoring email"
else
writeLog 'task' "Restoring email"
fi
# restore email messages
doRestore "${COMPOSE_PROJECT_NAME}_vmail-vol-1" "$dockerVolumeMail"; ec="$?"
case "$ec" in
0)
if [ "$verbose" -eq 1 ]; then
writeLog 'success' "Email messages restored"
else
writeLog 'done'
fi
;;
1)
if [ "$verbose" -eq 1 ]; then
writeLog 'error' '52' "There was an error restoring one or more email messages."
else
writeLog 'done' 'error'
writeLog 'error' '52' "There was an error restoring one or more email messages."
fi
;;
2)
if [ "$verbose" -eq 1 ]; then
writeLog 'error' '51' "Cannot locate email message backups!"
else
writeLog 'done' 'error'
writeLog 'error' '51' "Cannot locate email message backups!"
fi
;;
esac
# restore encryption key
doRestore "${COMPOSE_PROJECT_NAME}_crypt-vol-1" "$dockerVolumeCrypt"; ec="$?"
case "$ec" in
0)
if [ "$verbose" -eq 1 ]; then
writeLog 'success' "Encryption key restored"
else
writeLog 'done'
fi
;;
1)
if [ "$verbose" -eq 1 ]; then
writeLog 'error' '52' "There was an error restoring the encryption key! Any restored messages are likely *not* readable!"
else
writeLog 'done' 'error'
writeLog 'error' '52' "There was an error restoring the encryption key! Any restored messages are likely *not* readable!"
fi
;;
2)
if [ "$verbose" -eq 1 ]; then
writeLog 'error' '51' "Cannot locate encryption key backup!"
else
writeLog 'done' 'error'
writeLog 'error' '51' "Cannot locate encryption key backup!"
fi
;;
esac
fi
### restore postfix
if [ "$restorePostfix" -eq 1 ]; then
if [ "$verbose" -eq 1 ]; then
writeLog 'info' "Restoring postfix files"
else
writeLog 'task' "Restoring postfix files"
fi
doRestore "${COMPOSE_PROJECT_NAME}_postfix-vol-1" "$dockerVolumePostfix"; ec="$?"
case "$ec" in
0)
if [ "$verbose" -eq 1 ]; then
writeLog 'success' "Postfix files restored"
else
writeLog 'done'
fi
;;
1)
if [ "$verbose" -eq 1 ]; then
writeLog 'error' '52' "There was an error restoring one or more postfix files."
else
writeLog 'done' 'error'
writeLog 'error' '52' "There was an error restoring one or more postfix files."
fi
;;
2)
if [ "$verbose" -eq 1 ]; then
writeLog 'error' '51' "Cannot locate postfix backups!"
else
writeLog 'done' 'error'
writeLog 'error' '51' "Cannot locate postfix backups!"
fi
;;
esac
fi
### restore rspamd
if [ "$restoreRspamd" -eq 1 ]; then
if [ "$verbose" -eq 1 ]; then
writeLog 'info' "Restoring Rspamd files"
else
writeLog 'task' "Restoring Rspamd files"
fi
doRestore "${COMPOSE_PROJECT_NAME}_rspamd-vol-1" "$dockerVolumeRspamd"; ec="$?"
case "$ec" in
0)
if [ "$verbose" -eq 1 ]; then
writeLog 'success' "Rspamd files restored"
else
writeLog 'done'
fi
;;
1)
if [ "$verbose" -eq 1 ]; then
writeLog 'error' '52' "There was an error restoring one or more Rspamd files."
else
writeLog 'done' 'error'
writeLog 'error' '52' "There was an error restoring one or more Rspamd files."
fi
;;
2)
if [ "$verbose" -eq 1 ]; then
writeLog 'error' '51' "Cannot locate Rspamd backups!"
else
writeLog 'done' 'error'
writeLog 'error' '51' "Cannot locate Rspamd backups!"
fi
;;
esac
fi
### restore redis
if [ "$restoreRedis" -eq 1 ]; then
if [ "$verbose" -eq 1 ]; then
writeLog 'info' "Restoring redis database"
else
writeLog 'task' "Restoring redis database"
fi
doRestore "${COMPOSE_PROJECT_NAME}_redis-vol-1" "$dockerVolumeRedis"; ec="$?"
case "$ec" in
0)
if [ "$verbose" -eq 1 ]; then
writeLog 'success' "Redis database restored"
else
writeLog 'done'
fi
;;
1)
if [ "$verbose" -eq 1 ]; then
writeLog 'error' '52' "There was an error restoring the redis database. This is usually *not* a serious issue."
else
writeLog 'done' 'error'
writeLog 'error' '52' "There was an error restoring the redis database. This is usually *not* a serious issue."
fi
;;
2)
if [ "$verbose" -eq 1 ]; then
writeLog 'error' '51' "Cannot locate redis database backups!"
else
writeLog 'done' 'error'
writeLog 'error' '51' "Cannot locate redis database backups!"
fi
;;
esac
fi
### restart mailcow
writeLog 'task' "Starting mailcow"
if ! docker-compose up -d > /dev/null 2>&1; then
writeLog 'done' 'warn'
writeLog 'warn' '21' "Unable to automatically start mailcow containers. Please attempt a manual start and note any errors."
warnCount=$((warnCount+1))
fi
writeLog 'done'
### exit gracefully
writeLog 'success' "All processes completed"
printf "%s[%s] --- %s execution completed ---\n%s" "$magenta" "$(stamp)" "$scriptName" "$norm" >>"$logfile"
# note non-terminating errors
if [ "$errorCount" -gt 0 ]; then
printf "%s%s errors encountered!%s\n" "$err" "$errorCount" "$norm" >>"$logfile"
# note non-terminating errors
printf "%s[%s] --- %s execution completed with %s error(s) ---\n%s" "$err" "$(stamp)" "$scriptName" "$errorCount" "$norm" >> "$logfile"
exit 98
elif [ "$warnCount" -gt 0 ]; then
printf "%s[%s] --- %s execution completed with %s warning(s) ---\n%s" "$yellow" "$(stamp)" "$scriptName" "$warnCount" "$norm" >> "$logfile"
exit 97
else
writeLog 'success' "All processes completed"
printf "%s[%s] --- %s execution completed ---\n%s" "$magenta" "$(stamp)" "$scriptName" "$norm" >> "$logfile"
exit 0
fi
# note warnings
if [ "$warnCount" -gt 0 ]; then
printf "%s%s warnings issued!%s\n" "$yellow" "$warnCount" "$norm" >>"$logfile"
fi
exit 0
### error codes:
# 1: parameter error
@@ -370,9 +633,14 @@ exit 0
# 11: cannot locate SQL dump in backup directory
# 12: cannot start mysql-mailcow container
# 13: restoring SQL dump was unsuccessful
# 2x: Docker/Docker-Compose errors
# 20: cannot bring docker container(s) down successfully
# 21: cannot bring docker container(s) up successfully
# 5x: File restore errors
# 51: cannot locate source files in backup directory
# 52: error restoring one or more files
# 97: script completed with 1 or more warnings
# 98: script completed with 1 or more non-terminating errors
# 99: TERM signal trapped
# 100: could not change to mailcow-dockerized directory
# 101: could not stop container(s)
# 102: could not start container(s)
#EOF