Compare commits

...

3 Commits

Author SHA1 Message Date
Asif Bacchus
276de6d890 style(script): reformat and fix typos
- reformat spacing, indentation
- fix spelling mistakes
- change textblock functions to camelCase
- add shellcheck exceptions
2021-05-15 10:09:15 -06:00
Asif Bacchus
a4221ae6e4 refactor(script): used fixed export location
- previous random name made full restore virtually impossible
- use one container directory to make full restore easy
- store each backup in timestamped subdirectory for easy
incremental restore
2021-05-15 09:59:42 -06:00
Asif Bacchus
f87a2356a5 chore: update readme, remove vscode dir 2021-05-15 09:50:56 -06:00
8 changed files with 289 additions and 255 deletions

View File

@ -0,0 +1,13 @@
# Default ignored files
/shelf/
/workspace.xml
# Rider ignored files
/modules.xml
/contentModel.xml
/.idea.pilerBackup.iml
/projectSettingsUpdater.xml
# Datasource local storage ignored files
/../../../../../../../../../:\Redirected\Asif\Documents\RiderProjects\pilerBackup\.idea\.idea.pilerBackup.dir\.idea/dataSources/
/dataSources.local.xml
# Editor-based HTTP Client requests
/httpRequests/

View File

@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectDictionaryState">
<dictionary name="asif" />
</component>
</project>

View File

@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ContentModelUserStore">
<attachedFolders />
<explicitIncludes />
<explicitExcludes />
</component>
</project>

View File

@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="RIDER_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$/../.." />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>

View File

@ -0,0 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CommitMessageInspectionProfile">
<profile version="1.0">
<inspection_tool class="BodyLimit" enabled="true" level="ERROR" enabled_by_default="true" />
<inspection_tool class="SubjectBodySeparation" enabled="true" level="ERROR" enabled_by_default="true" />
<inspection_tool class="SubjectLimit" enabled="true" level="ERROR" enabled_by_default="true" />
</profile>
</component>
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>

View File

@ -1,3 +0,0 @@
{
"bookmarks": []
}

View File

@ -1,3 +1,3 @@
# pilerBackup
Use pilerexport to output raw EML files and then back them up to rsync using borg
Use pilerexport to output raw EML files and then back them up using borgbackup.

View File

@ -4,10 +4,9 @@
### Backup exported email from piler (bare-metal)
#######
### set environment variables
if [ -z "$TERM" ]; then
export TERM=dumb;
export TERM=dumb
fi
export PATH=$PATH:/usr/local/bin
@ -34,11 +33,9 @@ else
yellow=""
fi
### trap
trap trapExit 1 2 3 6
### functions
# bad configuration value passed in details file
@ -76,6 +73,7 @@ cleanup () {
# cleanup 503 if copied
if [ "$err503Copied" -eq 1 ]; then
if ! rm -f "$webroot/$err503File" 2>>"$logFile"; then
# shellcheck disable=SC2129
printf "%s[%s] -- [WARNING] Could not remove 503 error page." \
"$warn" "$(stamp)" >>"$logFile"
printf " Web interface will not function until this file is " \
@ -89,6 +87,7 @@ cleanup () {
fi
# remove EML temporary directory
if ! rm -rf "$exportDir" 2>>"$logFile"; then
# shellcheck disable=SC2129
printf "%s[%s] -- [WARNING] Could not remove EML export tmp dir:" \
"$warn" "$(stamp)" >>"$logFile"
printf "\n%s\n" "$exportDir" >>"$logFile"
@ -116,56 +115,56 @@ exitError () {
scriptHelp() {
newline
printf "%sUsage: %s [parameters]%s\n\n" "$bold" "$scriptName" "$norm"
textblock "There are no mandatory parameters. If a parameter is not supplied, its default value will be used. In the case of a switch parameter, it will remain DEactivated by default."
textBlock "There are no mandatory parameters. If a parameter is not supplied, its default value will be used. In the case of a switch parameter, it will remain DEACTIVATED by default."
newline
textblock "Parameters are listed then followed by a description of their effect on the following line. Finally, if a default value exists, it will be listed on the next line in (parentheses)."
textBlock "Parameters are listed then followed by a description of their effect on the following line. Finally, if a default value exists, it will be listed on the next line in (parentheses)."
newline
textblock "${magenta}--- script related ---${norm}"
textBlock "${magenta}--- script related ---${norm}"
newline
switchTextblock "-c | --config | --details"
textblock "Path to the configuration key/value pair file for this script."
defaultsTextblock "(scriptPath/scriptName.details)"
textBlockSwitch "-c | --config | --details"
textBlock "Path to the configuration key/value pair file for this script."
textBlockDefaults "(scriptPath/scriptName.details)"
newline
switchTextblock "-h | -? | --help"
textblock "This help screen."
textBlockSwitch "-h | -? | --help"
textBlock "This help screen."
newline
switchTextblock "-l | --log"
textblock "Path to write log file."
defaultsTextblock "(scriptPath/scriptName.log)"
textBlockSwitch "-l | --log"
textBlock "Path to write log file."
textBlockDefaults "(scriptPath/scriptName.log)"
newline
switchTextblock "[SWITCH] -v | --verbose"
textblock "Log borg output with increased verbosity (list all files). Careful! Your log file can get very large very quickly!"
defaultsTextblock "(normal output, option is OFF)"
textBlockSwitch "[SWITCH] -v | --verbose"
textBlock "Log borg output with increased verbosity (list all files). Careful! Your log file can get very large very quickly!"
textBlockDefaults "(normal output, option is OFF)"
newline
textblock "${magenta}--- export options ---${norm}"
textBlock "${magenta}--- export options ---${norm}"
newline
switchTextblock "[SWITCH] -a | --all"
textblock "Export ALL email saved by piler. This is generally only useful for an initial backup. Setting this switch will override any specified start/end dates."
defaultsTextblock "(use start/end dates, option is OFF)"
textBlockSwitch "[SWITCH] -a | --all"
textBlock "Export ALL email saved by piler. This is generally only useful for an initial backup. Setting this switch will override any specified start/end dates."
textBlockDefaults "(use start/end dates, option is OFF)"
newline
switchTextblock "--start"
textblock "Export email starting from this date (inclusive). Date MUST be provided in YYYY.MM.DD format."
defaultsTextblock "(yesterday's date)"
textBlockSwitch "--start"
textBlock "Export email starting from this date (inclusive). Date MUST be provided in YYYY.MM.DD format."
textBlockDefaults "(yesterday's date)"
newline
switchTextblock "--end"
textblock "Export email ending at this date (inclusive). Date MUST be provided in YYYY.MM.DD format."
defaultsTextblock "(yesterday's date)"
textBlockSwitch "--end"
textBlock "Export email ending at this date (inclusive). Date MUST be provided in YYYY.MM.DD format."
textBlockDefaults "(yesterday's date)"
newline
textblock "${magenta}--- 503 functionality ---${norm}"
textBlock "${magenta}--- 503 functionality ---${norm}"
newline
switchTextblock "[SWITCH] -5 | --use-503"
textblock "Copy an 'error 503' page/indicator file to your webroot for your webserver to find. Specifying this option will enable other 503 options."
defaultsTextblock "(do NOT copy, option is OFF)"
textBlockSwitch "[SWITCH] -5 | --use-503"
textBlock "Copy an 'error 503' page/indicator file to your webroot for your webserver to find. Specifying this option will enable other 503 options."
textBlockDefaults "(do NOT copy, option is OFF)"
newline
switchTextblock "--503-path"
textblock "Path to the file you want copied to your webroot as the 'error 503' page."
defaultsTextblock "(scriptPath/503_backup.html)"
textBlockSwitch "--503-path"
textBlock "Path to the file you want copied to your webroot as the 'error 503' page."
textBlockDefaults "(scriptPath/503_backup.html)"
newline
switchTextblock "-w | --webroot"
textblock "Path to where the 'error 503' file should be copied."
defaultsTextblock "(/usr/share/nginx/html/)"
textBlockSwitch "-w | --webroot"
textBlock "Path to where the 'error 503' file should be copied."
textBlockDefaults "(/usr/share/nginx/html/)"
newline
textblock "More details and examples of script usage can be found in the repo wiki at ${yellow}https://git.asifbacchus.app/asif/pilerBackup/wiki${norm}"
textBlock "More details and examples of script usage can be found in the repo wiki at ${yellow}https://git.asifbacchus.app/asif/pilerBackup/wiki${norm}"
newline
}
@ -174,15 +173,15 @@ stamp () {
(date +%F" "%T)
}
textblock() {
textBlock() {
printf "%s\n" "$1" | fold -w "$width" -s
}
defaultsTextblock() {
textBlockDefaults() {
printf "%s%s%s\n" "$yellow" "$1" "$norm"
}
switchTextblock() {
textBlockSwitch() {
printf "%s%s%s\n" "$cyan" "$1" "$norm"
}
@ -203,7 +202,6 @@ trapExit () {
### end of functions
### default variable values
## script related
@ -233,7 +231,6 @@ exportStart=$( date -d 'yesterday' +'%Y.%m.%d' )
exportEnd=$(date -d 'yesterday' +'%Y.%m.%d')
pilerUser='piler'
### process startup parameters
while [ $# -gt 0 ]; do
case "$1" in
@ -344,10 +341,9 @@ while [ $# -gt 0 ]; do
shift
done
### check pre-requisites and default values
# check if running as root, otherwise exit
if [ $( id -u ) -ne 0 ]; then
if [ "$(id -u)" -ne 0 ]; then
printf "\n%sERROR: script MUST be run as ROOT%s\n\n" "$err" "$norm"
exit 2
fi
@ -375,21 +371,18 @@ if [ "$use503" -eq 1 ]; then
fi
fi
### start logging
printf "%s[%s] --- Start %s execution ---%s\n" \
"$magenta" "$(stamp)" "$scriptName" "$norm" >>"$logFile"
printf "%s[%s] -- [INFO] Log located at %s%s%s --%s\n" \
"$cyan" "$(stamp)" "$yellow" "$logFile" "$cyan" "$norm" >>"$logFile"
### 503 functionality
if [ "$use503" -eq 1 ]; then
printf "%s[%s] -- [INFO] Copying 503 error page to " \
"$cyan" "$(stamp)" >>"$logFile"
printf "webroot -- %s\n" "$norm" >>"$logFile"
if ! \cp --force "${err503Path}" "${webroot}/${err503File}" 2>> "$logFile"
then
if ! \cp --force "${err503Path}" "${webroot}/${err503File}" 2>>"$logFile"; then
printf "%s[%s] -- [WARNING] Failed to copy 503 error page. " \
"$warn" "$(stamp)" >>"$logFile"
printf "Web users will NOT be notified --%s\n" "$norm" >>"$logFile"
@ -402,35 +395,39 @@ if [ "$use503" -eq 1 ]; then
fi
fi
### read details file to get variables needed to dump sql and run borg
# check if config details file was provided as a relative or absolute path
case "${configDetails}" in
/*)
# absolute path, no need to rewrite variable
# shellcheck source=pilerbackup.details
. "${configDetails}"
;;
*)
# relative path, prepend './' to create absolute path
# shellcheck source=pilerbackup.details
. "./${configDetails}"
;;
esac
printf "%s[%s] -- [INFO] %s%s%s imported --%s\n" \
"$cyan" "$(stamp)" "$yellow" "$configDetails" "$cyan" "$norm" >>"$logFile"
## create tmp directory and change to it for export operations
# create temporary directory to dump exported email from piler
if ! exportDir=$( mktemp -d 2>>"$logFile" ); then
tmpdir="/tmp/emailbackup/$(date +%F_%T)"
if ! exportDir=$(mkdir -p "${tmpdir}" 2>>"$logFile"); then
exitError 111 "Could not create temporary directory for exported EML files"
fi
# grant pilerUser permission to write to temporary directory
if ! (chown root:${pilerUser} "$exportDir" && chmod 770 "$exportDir"); then
if # shellcheck disable=SC2086
! (chown root:${pilerUser} "$exportDir" && chmod 770 "$exportDir")
then
exitError 112 "Could not set permissions on temporary directory"
fi
if ! cd "$exportDir"; then
exitError 113 "Unable to change to temporary export directory"
fi
# shellcheck disable=SC2129
printf "%s[%s] -- [INFO] EML files will be temporarily stored in:" \
"$cyan" "$(stamp)" >>"$logFile"
printf "\n\t%s%s/%s --%s\n" "$yellow" "$exportDir" "$cyan" "$norm" >>"$logFile"
@ -452,12 +449,10 @@ else
"$ok" "$(stamp)" "$norm" >>"$logFile"
fi
### pre-backup tasks completed -- move to borg tasks
printf "%s[%s] -- [SUCCESS] Pre-backup tasks completed --%s\n" \
"$ok" "$(stamp)" "$norm" >>"$logFile"
### Run borg variable checks
printf "%s[%s] -- [INFO] Verifying supplied borg details --%s\n" \
"$cyan" "$(stamp)" "$norm" >>"$logFile"
@ -542,7 +537,7 @@ printf "%sdetails:borgXtraListPath%s -- %s[OK]%s\n" \
includeXtra=1
fi
## check if exlusion list file is specified
## check if exclusion list file is specified
if [ -n "${borgExcludeListPath}" ]; then
# check if the file actually exists
if [ ! -f "${borgExcludeListPath}" ]; then
@ -551,7 +546,6 @@ if [ -n "${borgExcludeListPath}" ]; then
exclusions=1
fi
### create borg temp dir:
## python requires a writable temporary directory when unpacking borg and
## executing commands. This defaults to /tmp but many systems mount /tmp with
@ -570,7 +564,6 @@ if [ ! -d "${borgBaseDir}/tmp" ]; then
fi
export TMPDIR="${borgBaseDir}/tmp"
### execute borg depending on whether extra files and/or exclusions are defined
## construct the proper borg commandline
@ -617,11 +610,11 @@ else
warnCount=$((warnCount + 1))
fi
### execute borg prune if paramters are provided, otherwise skip with a warning
### execute borg prune if parameters are provided, otherwise skip with a warning
if [ -n "${borgPruneSettings}" ]; then
printf "%s[%s] -- [INFO] Executing borg prune operation --%s\n" \
"$cyan" "$(stamp)" "$norm" >>"$logFile"
# shellcheck disable=SC2086
borg prune --show-rc -v ${borgPruneParams} ${borgPruneSettings} \
2>>"$logFile"
borgPruneResult="$?"
@ -656,7 +649,6 @@ if [ -n "${borgPruneResult}" ]; then
fi
fi
### all processes successfully completed, cleanup and exit gracefully
# note successful completion of borg commands
@ -678,8 +670,6 @@ else
fi
exit 0
### error codes
# 1: parameter error
# 2: not run as root