#!/bin/sh ####### ### Backup exported email from piler (bare-metal) ####### ### set environment variables if [ -z "$TERM" ]; then export TERM=dumb fi export PATH=$PATH:/usr/local/bin ### text formatting presents if command -v tput >/dev/null; then bold=$(tput bold) cyan=$(tput setaf 6) err=$(tput bold)$(tput setaf 1) magenta=$(tput setaf 5) norm=$(tput sgr0) ok=$(tput setaf 2) warn=$(tput bold)$(tput setaf 3) width=$(tput cols) yellow=$(tput setaf 3) else bold="" cyan="" err="" magenta="" norm="" ok="" warn="" width=80 yellow="" fi ### trap trap trapExit 1 2 3 6 ### functions # bad configuration value passed in details file badDetails() { if [ "$1" = "empty" ]; then exitError 130 "details:${2} cannot be NULL (undefined)" elif [ "$1" = "dne" ]; then exitError 131 "details:${2} file or directory does not exist." fi } # bad parameter passed to script badParam() { if [ "$1" = "dne" ]; then printf "\n%sError: '%s %s'\n" "$err" "$2" "$3" printf "file or directory does not exist.%s\n\n" "$norm" exit 1 elif [ "$1" = "empty" ]; then printf "\n%sError: '%s' cannot have a NULL (empty) value.\n" "$err" "$2" printf "%sPlease use '--help' for assistance%s\n\n" "$cyan" "$norm" exit 1 elif [ "$1" = "svc" ]; then printf "\n%sError: '%s %s': Service does not exist!${norm}\n\n" \ "$err" "$2" "$3" exit 1 elif [ "$1" = "user" ]; then printf "\n%sError: '%s %s': User does not exist!${norm}\n\n" \ "$err" "$2" "$3" exit 1 fi } # cleanup cleanup() { # cleanup 503 if copied if [ "$err503Copied" -eq 1 ]; then if ! rm -f "$webroot/$err503File" 2>>"$logFile"; then # shellcheck disable=SC2129 printf "%s[%s] -- [WARNING] Could not remove 503 error page." \ "$warn" "$(stamp)" >>"$logFile" printf " Web interface will not function until this file is " \ >>"$logFile" printf "removed --%s\n" "$norm" >>"$logFile" warnCount=$((warnCount + 1)) else printf "%s[%s] -- [INFO] 503 error page removed --%s\n" \ "$cyan" "$(stamp)" "$norm" >>"$logFile" fi fi # remove EML temporary directory if ! rm -rf "$exportDir" 2>>"$logFile"; then # shellcheck disable=SC2129 printf "%s[%s] -- [WARNING] Could not remove EML export tmp dir:" \ "$warn" "$(stamp)" >>"$logFile" printf "\n%s\n" "$exportDir" >>"$logFile" printf "please remove this directory manually! --%s\n" "$norm" \ >>"$logFile" warnCount=$((warnCount + 1)) else printf "%s[%s] -- [INFO] EML temp directory removed --%s\n" \ "$cyan" "$(stamp)" "$norm" >>"$logFile" fi } # call cleanup and then exit with error report exitError() { printf "%s[%s] -- [ERROR] %s: %s --%s\n" \ "$err" "$(stamp)" "$1" "$2" "$norm" >>"$logFile" cleanup # note script completion with error printf "%s[%s] --- %s execution completed with error ---%s\n" \ "$err" "$(stamp)" "$scriptName" "$norm" >>"$logFile" exit "$1" } # display script help information scriptHelp() { newline printf "%sUsage: %s [parameters]%s\n\n" "$bold" "$scriptName" "$norm" textBlock "There are no mandatory parameters. If a parameter is not supplied, its default value will be used. In the case of a switch parameter, it will remain DEACTIVATED by default." newline textBlock "Parameters are listed then followed by a description of their effect on the following line. Finally, if a default value exists, it will be listed on the next line in (parentheses)." newline textBlock "${magenta}--- script related ---${norm}" newline textBlockSwitch "-c | --config | --details" textBlock "Path to the configuration key/value pair file for this script." textBlockDefaults "(scriptPath/scriptName.details)" newline textBlockSwitch "-h | -? | --help" textBlock "This help screen." newline textBlockSwitch "-l | --log" textBlock "Path to write log file." textBlockDefaults "(scriptPath/scriptName.log)" newline textBlockSwitch "[SWITCH] -v | --verbose" textBlock "Log borg output with increased verbosity (list all files). Careful! Your log file can get very large very quickly!" textBlockDefaults "(normal output, option is OFF)" newline textBlock "${magenta}--- export options ---${norm}" newline textBlockSwitch "[SWITCH] -a | --all" textBlock "Export ALL email saved by piler. This is generally only useful for an initial backup. Setting this switch will override any specified start/end dates." textBlockDefaults "(use start/end dates, option is OFF)" newline textBlockSwitch "--start" textBlock "Export email starting from this date (inclusive). Date MUST be provided in YYYY.MM.DD format." textBlockDefaults "(yesterday's date)" newline textBlockSwitch "--end" textBlock "Export email ending at this date (inclusive). Date MUST be provided in YYYY.MM.DD format." textBlockDefaults "(yesterday's date)" newline textBlock "${magenta}--- 503 functionality ---${norm}" newline textBlockSwitch "[SWITCH] -5 | --use-503" textBlock "Copy an 'error 503' page/indicator file to your webroot for your webserver to find. Specifying this option will enable other 503 options." textBlockDefaults "(do NOT copy, option is OFF)" newline textBlockSwitch "--503-path" textBlock "Path to the file you want copied to your webroot as the 'error 503' page." textBlockDefaults "(scriptPath/503_backup.html)" newline textBlockSwitch "-w | --webroot" textBlock "Path to where the 'error 503' file should be copied." textBlockDefaults "(/usr/share/nginx/html/)" newline textBlock "More details and examples of script usage can be found in the repo wiki at ${yellow}https://git.asifbacchus.app/asif/pilerBackup/wiki${norm}" newline } # generate dynamic timestamps stamp() { (date +%F" "%T) } textBlock() { printf "%s\n" "$1" | fold -w "$width" -s } textBlockDefaults() { printf "%s%s%s\n" "$yellow" "$1" "$norm" } textBlockSwitch() { printf "%s%s%s\n" "$cyan" "$1" "$norm" } newline() { printf "\n" } # same as exitError but for signal captures trapExit() { printf "%s[%s] -- [ERROR] 99: Caught signal --%s\n" \ "$err" "$(stamp)" "$norm" >>"$logFile" cleanup # note script completion with error printf "%s[%s] --- %s execution was terminated via signal ---%s\n" \ "$err" "$(stamp)" "$scriptName" "$norm" >>"$logFile" exit 99 } ### end of functions ### default variable values ## script related # store logfile in the same directory as this script file using the same file # name as the script but with the extension '.log' scriptPath="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)" scriptName="$(basename "$0")" logFile="$scriptPath/${scriptName%.*}.log" warnCount=0 configDetails="$scriptPath/${scriptName%.*}.details" err503Copied=0 includeXtra=0 exclusions=0 # borg output verbosity -- normal borgCreateParams='--stats' borgPruneParams='--list' # 503 related use503=0 err503Path="$scriptPath/503_backup.html" err503File="${err503Path##*/}" webroot="/usr/share/nginx/html" # piler-export related exportAll=0 exportStart=$(date -d 'yesterday' +'%Y.%m.%d') exportEnd=$(date -d 'yesterday' +'%Y.%m.%d') pilerUser='piler' ### process startup parameters while [ $# -gt 0 ]; do case "$1" in -h | -\? | --help) # display help scriptHelp exit 0 ;; -l | --log) # set log file location if [ -n "$2" ]; then logFile="${2%/}" shift else badParam empty "$@" fi ;; -v | --verbose) # set verbose logging from borg borgCreateParams='--list --stats' borgPruneParams='--list' ;; -c | --config | --details) # location of config details file if [ -n "$2" ]; then if [ -f "$2" ]; then configDetails="${2%/}" shift else badParam dne "$@" fi else badParam empty "$@" fi ;; -5 | --use-503) # enable copying 503 error page to webroot use503=1 ;; --503-path) # FULL path to 503 file if [ -n "$2" ]; then if [ -f "$2" ]; then err503Path="${2%/}" err503File="${2##*/}" shift else badParam dne "$@" fi else badParam empty "$@" fi ;; -w | --webroot) # path to webroot (copy 503) if [ -n "$2" ]; then if [ -d "$2" ]; then webroot="${2%/}" shift else badParam dne "$@" fi else badParam empty "$@" fi ;; -a | --all) # export ALL email stored in piler exportAll=1 ;; --start) # export starting at this date if [ -n "$2" ]; then exportStart="$2" shift else badParam empty "$@" fi ;; --end) # export ending at this date if [ -n "$2" ]; then exportEnd="$2" shift else badParam empty "$@" fi ;; --user) # specify piler user account name if [ -n "$2" ]; then if id ${pilerUser} >/dev/null 2>&1; then pilerUser="$2" shift else badParam user "$@" fi else badParam empty "$@" fi ;; *) printf "%s\nUnknown option: %s\n" "$err" "$1" printf "%sUse '--help' for valid options.%s\n\n" "$cyan" "$norm" exit 1 ;; esac shift done ### check pre-requisites and default values # check if running as root, otherwise exit if [ "$(id -u)" -ne 0 ]; then printf "\n%sERROR: script MUST be run as ROOT%s\n\n" "$err" "$norm" exit 2 fi # find pilerexport, otherwise this is all pointless pilerExport=$(command -v pilerexport) if [ ! "$pilerExport" ]; then printf "\n%sERROR: cannot find 'pilerexport'%s\n\n" "$err" "$norm" exit 4 fi # does the details file exist? if [ ! -f "$configDetails" ]; then badParam dne "(--details default)" "$configDetails" fi # is borg installed? if ! command -v borg >/dev/null; then printf "\n%sERROR: BORG is not installed on this system!%s\n\n" "$err" "$norm" exit 3 fi # if 503 functionality is enabled, do 503 related files exist? if [ "$use503" -eq 1 ]; then if [ ! -f "$err503Path" ]; then badParam dne "(--503-path default)" "$err503Path" elif [ ! -d "$webroot" ]; then badParam dne "(--webroot default)" "$webroot" fi fi ### start logging printf "%s[%s] --- Start %s execution ---%s\n" \ "$magenta" "$(stamp)" "$scriptName" "$norm" >>"$logFile" printf "%s[%s] -- [INFO] Log located at %s%s%s --%s\n" \ "$cyan" "$(stamp)" "$yellow" "$logFile" "$cyan" "$norm" >>"$logFile" ### 503 functionality if [ "$use503" -eq 1 ]; then printf "%s[%s] -- [INFO] Copying 503 error page to " \ "$cyan" "$(stamp)" >>"$logFile" printf "webroot -- %s\n" "$norm" >>"$logFile" if ! \cp --force "${err503Path}" "${webroot}/${err503File}" 2>>"$logFile"; then printf "%s[%s] -- [WARNING] Failed to copy 503 error page. " \ "$warn" "$(stamp)" >>"$logFile" printf "Web users will NOT be notified --%s\n" "$norm" >>"$logFile" warnCount=$((warnCount + 1)) else printf "%s[%s] -- [SUCCESS] 503 error page copied --%s\n" \ "$ok" "$(stamp)" "$norm" >>"$logFile" # set cleanup flag err503Copied=1 fi fi ### read details file to get variables needed to dump sql and run borg # check if config details file was provided as a relative or absolute path case "${configDetails}" in /*) # absolute path, no need to rewrite variable # shellcheck source=pilerbackup.details . "${configDetails}" ;; *) # relative path, prepend './' to create absolute path # shellcheck source=pilerbackup.details . "./${configDetails}" ;; esac printf "%s[%s] -- [INFO] %s%s%s imported --%s\n" \ "$cyan" "$(stamp)" "$yellow" "$configDetails" "$cyan" "$norm" >>"$logFile" ## create tmp directory and change to it for export operations # create temporary directory to dump exported email from piler tmpdir="/tmp/emailbackup/$(date +%F_%T)" if ! exportDir=$(mkdir -p "${tmpdir}" 2>>"$logFile"); then exitError 111 "Could not create temporary directory for exported EML files" fi # grant pilerUser permission to write to temporary directory if # shellcheck disable=SC2086 ! (chown root:${pilerUser} "$exportDir" && chmod 770 "$exportDir") then exitError 112 "Could not set permissions on temporary directory" fi if ! cd "$exportDir"; then exitError 113 "Unable to change to temporary export directory" fi # shellcheck disable=SC2129 printf "%s[%s] -- [INFO] EML files will be temporarily stored in:" \ "$cyan" "$(stamp)" >>"$logFile" printf "\n\t%s%s/%s --%s\n" "$yellow" "$exportDir" "$cyan" "$norm" >>"$logFile" ## export EML from piler printf "%s[%s] -- [INFO] Exporting EML files from piler --%s\n" \ "$cyan" "$(stamp)" "$norm" >>"$logFile" if [ "$exportAll" -eq 1 ]; then if ! ${pilerExport} -A 2>>"$logFile"; then exitError 115 "There was a problem while exporting EML from piler" fi printf "%s[%s] -- [SUCCESS] Exported EML files from piler --%s\n" \ "$ok" "$(stamp)" "$norm" >>"$logFile" else if ! ${pilerExport} -a "$exportStart" -b "$exportEnd" 2>>"$logFile"; then exitError 115 "There was a problem while exporting EML from piler" fi printf "%s[%s] -- [SUCCESS] Exported EML files from piler --%s\n" \ "$ok" "$(stamp)" "$norm" >>"$logFile" fi ### pre-backup tasks completed -- move to borg tasks printf "%s[%s] -- [SUCCESS] Pre-backup tasks completed --%s\n" \ "$ok" "$(stamp)" "$norm" >>"$logFile" ### Run borg variable checks printf "%s[%s] -- [INFO] Verifying supplied borg details --%s\n" \ "$cyan" "$(stamp)" "$norm" >>"$logFile" ## verify borg base directory if [ -z "${borgBaseDir}" ]; then badDetails empty 'borgBaseDir' elif [ ! -d "${borgBaseDir}" ]; then badDetails dne 'borgBaseDir' fi printf "%sdetails:borgBaseDir%s -- %s[OK]%s\n" \ "$magenta" "$norm" "$ok" "$norm" >>"$logFile" export BORG_BASE_DIR="${borgBaseDir%/}" ## check path to SSH keyfile if [ -z "${borgSSHKey}" ]; then badDetails empty 'borgSSHKey' elif [ ! -f "${borgSSHKey}" ]; then badDetails dne 'borgSSHKey' fi printf "%sdetails:borgSSHKey%s -- %s[OK]%s\n" \ "$magenta" "$norm" "$ok" "$norm" >>"$logFile" export BORG_RSH="ssh -i ${borgSSHKey}" ## check borg repo connect string if [ -z "${borgConnectRepo}" ]; then badDetails empty 'borgConnectRepo' fi printf "%sdetails:borgConnectRepo%s -- %s[OK]%s\n" \ "$magenta" "$norm" "$ok" "$norm" >>"$logFile" export BORG_REPO="${borgConnectRepo}" ## check borg repo password if [ -n "${borgRepoPassphrase}" ]; then printf "%sdetails:borgRepoPassphrase%s -- %s[OK]%s\n" \ "$magenta" "$norm" "$ok" "$norm" >>"$logFile" export BORG_PASSPHRASE="${borgRepoPassphrase}" else # if passwd is blank intentionally, this is insecure printf "%s-- [WARNING] Using a borg repo without a password is an " \ "$warn" >>"$logFile" printf "insecure configuration --%s\n" "$norm" >>"$logFile" warnCount=$((warnCount + 1)) # if this was an accident, we need to provide a bogus passwd so borg fails # otherwise it will sit forever just waiting for input export BORG_PASSPHRASE="DummyPasswordSoBorgFails" fi ## check borg repository keyfile location if [ -z "${borgKeyfileLocation}" ]; then printf "%sdetails:borgKeyfileLocation %s-- %s[DEFAULT]%s\n" "$magenta" "$norm" "$ok" "$norm" >>"$logFile" else # check if keyfile location exists if [ ! -f "${borgKeyfileLocation}" ]; then badDetails dne 'borgKeyfileLocation' fi printf "%sdetails:borgKeyfileLocation %s-- %s[OK]%s\n" "$magenta" "$norm" "$ok" "$norm" >>"$logFile" export BORG_KEY_FILE="${borgKeyfileLocation}" fi ## export borg remote path, if specified if [ -n "${borgRemote}" ]; then export BORG_REMOTE_PATH="${borgRemote}"; fi ## read additional files if [ -n "${borgXtraListPath}" ]; then # check if file actually exists if [ ! -f "${borgXtraListPath}" ]; then badDetails dne 'borgXtraListPath' fi # read file contents into concatenated list for echo to cmdline while read -r xtraItem; do if [ -z "${xtraList}" ]; then xtraList="${xtraItem}" else xtraList="${xtraList} ${xtraItem}" fi done <>"$logFile" includeXtra=1 fi ## check if exclusion list file is specified if [ -n "${borgExcludeListPath}" ]; then # check if the file actually exists if [ ! -f "${borgExcludeListPath}" ]; then badDetails dne 'borgExcludeListPath' fi exclusions=1 fi ### create borg temp dir: ## python requires a writable temporary directory when unpacking borg and ## executing commands. This defaults to /tmp but many systems mount /tmp with ## the 'noexec' option for security. Thus, we will use/create a 'tmp' folder ## within the BORG_BASE_DIR and instruct python to use that instead of /tmp # check if BORG_BASE_DIR/tmp exists, if not, create it if [ ! -d "${borgBaseDir}/tmp" ]; then if ! mkdir "${borgBaseDir}/tmp"; then exitError 132 "Unable to create borg ${borgBaseDir}/tmp directory" else printf "%s[%s] -- [INFO] Created %s%s/tmp " \ "$cyan" "$(stamp)" "$yellow" "${borgBaseDir}" >>"$logFile" printf "%s--%s\n" "$cyan" "$norm" >>"$logFile" fi fi export TMPDIR="${borgBaseDir}/tmp" ### execute borg depending on whether extra files and/or exclusions are defined ## construct the proper borg commandline # base command if [ "$exclusions" -eq 0 ]; then borgCMD="borg --show-rc create ${borgCreateParams} \ ::$(date +%Y-%m-%d_%H%M%S) \ ${exportDir}" elif [ "$exclusions" -eq 1 ]; then borgCMD="borg --show-rc create ${borgCreateParams} \ --exclude-from ${borgExcludeListPath} \ ::$(date +%Y-%m-%d_%H%M%S) \ ${exportDir}" fi # add extra locations if defined if [ "$includeXtra" -eq 1 ]; then borgCMD="${borgCMD} ${xtraList}" fi # execute borg printf "%s[%s] -- [INFO] Executing borg backup operation --%s\n" \ "$cyan" "$(stamp)" "$norm" >>"$logFile" ${borgCMD} 2>>"$logFile" borgResult="$?" ## check borg exit status if [ "$borgResult" -eq 0 ]; then printf "%s[%s] -- [SUCCESS] Borg backup completed --%s\n" \ "$ok" "$(stamp)" "$norm" >>"$logFile" elif [ "$borgResult" -eq 1 ]; then printf "%s[%s] -- [WARNING] Borg completed with warnings. " \ "$warn" "$(stamp)" >>"$logFile" printf "Review this logfile for details --%s\n" "$norm" >>"$logFile" warnCount=$((warnCount + 1)) elif [ "$borgResult" -ge 2 ]; then err_1="Borg exited with a critical error. Please review this log file" err_2="for details." exitError 138 "$err_1 $err_2" else printf "%s[%s] -- [WARNING] Borg exited with unknown return code. " \ "$warn" "$(stamp)" >>"$logFile" printf "Review this logfile for details --%s\n" "$norm" >>"$logFile" warnCount=$((warnCount + 1)) fi ### execute borg prune if parameters are provided, otherwise skip with a warning if [ -n "${borgPruneSettings}" ]; then printf "%s[%s] -- [INFO] Executing borg prune operation --%s\n" \ "$cyan" "$(stamp)" "$norm" >>"$logFile" # shellcheck disable=SC2086 borg prune --show-rc -v ${borgPruneParams} ${borgPruneSettings} \ 2>>"$logFile" borgPruneResult="$?" else printf "%s[%s] -- [WARNING] No prune parameters provided. " \ "$warn" "$(stamp)" >>"$logFile" printf "Your archive will continue growing with each backup --%s\n" \ "$norm" >>"$logFile" warnCount=$((warnCount + 1)) fi ## report on prune operation if executed if [ -n "${borgPruneResult}" ]; then if [ "${borgPruneResult}" -eq 0 ]; then printf "%s[%s] -- [SUCCESS] Borg prune completed --%s\n" \ "$ok" "$(stamp)" "$norm" >>"$logFile" elif [ "$borgPruneResult" -eq 1 ]; then printf "%s[%s] -- [WARNING] Borg prune completed with warnings. " \ "$warn" "$(stamp)" >>"$logFile" printf "Review this logfile for details --%s\n" "$norm" >>"$logFile" warnCount=$((warnCount + 1)) elif [ "$borgPruneResult" -ge 2 ]; then err_1="Borg prune exited with a critical error. Please review this" err_2="log file for details." exitError 139 "$err_1 $err_2" else printf "%s[%s] -- [WARNING] Borg prune exited with an unknown " \ "$warn" "$(stamp)" >>"$logFile" printf "return code. Review this logfile for details --%s\n" \ "$norm" >>"$logFile" warnCount=$((warnCount + 1)) fi fi ### all processes successfully completed, cleanup and exit gracefully # note successful completion of borg commands printf "%s[%s] -- [SUCCESS] Backup operations completed --%s\n" \ "$ok" "$(stamp)" "$norm" >>"$logFile" # cleanup cleanup # note complete success, tally warnings and exit printf "%s[%s] -- [SUCCESS] All processes completed --%s\n" \ "$ok" "$(stamp)" "$norm" >>"$logFile" printf "%s[%s] --- %s execution completed ---%s\n" \ "$magenta" "$(stamp)" "$scriptName" "$norm" >>"$logFile" if [ "$warnCount" -gt 0 ]; then printf "%s%s warnings issued!%s\n" "$warn" "${warnCount}" "$norm" >>"$logFile" else printf "%s0 warnings issued.%s\n" "$ok" "$norm" >>"$logFile" fi exit 0 ### error codes # 1: parameter error # 2: not run as root # 3: borg not installed # 4: cannot find pilerexport executable # 99: TERM signal trapped # 111: could not create tmp dir for EML dump from piler-export # 112: could not set permissions on tmp dir # 113: could not change to tmp dir to start export operation # 115: problem during piler-export operation # 130: null configuration variable in details file # 131: invalid configuration variable in details file # 138: borg exited with a critical error # 139: borg prune exited with a critical error