Compare commits

...

33 Commits

Author SHA1 Message Date
Asif Bacchus 276de6d890 style(script): reformat and fix typos
- reformat spacing, indentation
- fix spelling mistakes
- change textblock functions to camelCase
- add shellcheck exceptions
2021-05-15 10:09:15 -06:00
Asif Bacchus a4221ae6e4 refactor(script): used fixed export location
- previous random name made full restore virtually impossible
- use one container directory to make full restore easy
- store each backup in timestamped subdirectory for easy
incremental restore
2021-05-15 09:59:42 -06:00
Asif Bacchus f87a2356a5 chore: update readme, remove vscode dir 2021-05-15 09:50:56 -06:00
Asif Bacchus 75368e108b update licence and export prefs 2020-08-14 00:44:51 -06:00
Asif Bacchus d0e8eca1d6 add keyfile to details file 2020-08-14 00:43:47 -06:00
Asif Bacchus b6ce0fc673 add keyfile support 2020-08-14 00:42:00 -06:00
Asif Bacchus 6fc82aac83 cleanup printf formatting 2020-08-14 00:39:23 -06:00
Asif Bacchus 760399a0fd update scriptHelp 2020-08-14 00:20:49 -06:00
Asif Bacchus 868398858e add defaults text formatting function 2020-08-14 00:10:48 -06:00
Asif Bacchus 44e57df831 func trapExit: cleanup printf formatting 2020-08-14 00:03:32 -06:00
Asif Bacchus bbea5e871c add textblock formatting functions 2020-08-14 00:02:36 -06:00
Asif Bacchus e74971997f funcs cleanup, exitError: printf formatting 2020-08-14 00:00:09 -06:00
Asif Bacchus ffc7579666 func badParam: proper printf formatting 2020-08-13 23:53:51 -06:00
Asif Bacchus acec0d6b23 check for tput before setting formatting 2020-08-13 23:51:36 -06:00
Asif Bacchus 5b1678c6ae set term to dummy for cron, update path 2020-08-13 23:48:52 -06:00
Asif Bacchus 3f08853826 update logwatch integration 2019-07-30 02:27:09 -06:00
Asif Bacchus c01ea25ed1 update sample logrotate config 2019-07-30 02:26:07 -06:00
Asif Bacchus 698c19b8ed update sample backup file locations 2019-07-30 02:25:17 -06:00
Asif Bacchus 702e039e11 fix errant filename 2019-07-26 06:01:44 -06:00
Asif Bacchus e495b65b24 check for root 2019-07-26 02:33:57 -06:00
Asif Bacchus ebc77e8137 test for pilerexport and use found full path 2019-07-26 02:29:05 -06:00
Asif Bacchus 5dff2a6919 logwatch configuration 2019-07-26 01:25:03 -06:00
Asif Bacchus 224ddd931d sample logrotate config 2019-07-26 01:15:29 -06:00
Asif Bacchus 034d12a85e new directory structure 2019-07-26 01:14:18 -06:00
Asif Bacchus 0de191f904 add simple 503 error page 2019-07-26 01:13:29 -06:00
Asif Bacchus 171cdbe676 clean up spacing errors for success msg in log 2019-07-25 13:10:59 -06:00
Asif Bacchus e273f3274c fix permissions command structure error 2019-07-25 13:06:49 -06:00
Asif Bacchus 8fb80564dd fix incorrect permission assignment on tmp dir 2019-07-25 12:57:15 -06:00
Asif Bacchus f12aa53e54 move export user notifications 2019-07-25 12:46:31 -06:00
Asif Bacchus 257667f12a grant pilerUser permission to write to tmp dir 2019-07-25 12:42:20 -06:00
Asif Bacchus 9c8c250318 change 'cd exportDir' error code 2019-07-25 12:42:02 -06:00
Asif Bacchus 649d162d4c add piler user param and err check 2019-07-25 12:30:11 -06:00
Asif Bacchus c3aa35ab15 fix error in tmp directory removal cmd 2019-07-25 12:16:09 -06:00
19 changed files with 957 additions and 649 deletions
+1
View File
@@ -73,3 +73,4 @@
.gitattributes export-ignore
.gitignore export-ignore
.vscode export-ignore
+13
View File
@@ -0,0 +1,13 @@
# Default ignored files
/shelf/
/workspace.xml
# Rider ignored files
/modules.xml
/contentModel.xml
/.idea.pilerBackup.iml
/projectSettingsUpdater.xml
# Datasource local storage ignored files
/../../../../../../../../../:\Redirected\Asif\Documents\RiderProjects\pilerBackup\.idea\.idea.pilerBackup.dir\.idea/dataSources/
/dataSources.local.xml
# Editor-based HTTP Client requests
/httpRequests/
+6
View File
@@ -0,0 +1,6 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ProjectDictionaryState">
<dictionary name="asif" />
</component>
</project>
+8
View File
@@ -0,0 +1,8 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="ContentModelUserStore">
<attachedFolders />
<explicitIncludes />
<explicitExcludes />
</component>
</project>
+7
View File
@@ -0,0 +1,7 @@
<?xml version="1.0" encoding="UTF-8"?>
<module type="RIDER_MODULE" version="4">
<component name="NewModuleRootManager">
<content url="file://$MODULE_DIR$/../.." />
<orderEntry type="sourceFolder" forTests="false" />
</component>
</module>
+13
View File
@@ -0,0 +1,13 @@
<?xml version="1.0" encoding="UTF-8"?>
<project version="4">
<component name="CommitMessageInspectionProfile">
<profile version="1.0">
<inspection_tool class="BodyLimit" enabled="true" level="ERROR" enabled_by_default="true" />
<inspection_tool class="SubjectBodySeparation" enabled="true" level="ERROR" enabled_by_default="true" />
<inspection_tool class="SubjectLimit" enabled="true" level="ERROR" enabled_by_default="true" />
</profile>
</component>
<component name="VcsDirectoryMappings">
<mapping directory="$PROJECT_DIR$" vcs="Git" />
</component>
</project>
-19
View File
@@ -1,19 +0,0 @@
{
"bookmarks": [
{
"fsPath": "$ROOTPATH$/pilerbackup.sh",
"bookmarks": [
-1,
201,
-1,
-1,
-1,
-1,
-1,
-1,
-1,
-1
]
}
]
}
+1 -1
View File
@@ -1,6 +1,6 @@
MIT License
Copyright (c) 2019 Asif Bacchus
Copyright (c) 2019-2020 Asif Bacchus
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
+1 -1
View File
@@ -1,3 +1,3 @@
# pilerBackup
Use pilerexport to output raw EML files and then back them up to rsync using borg
Use pilerexport to output raw EML files and then back them up using borgbackup.
+15
View File
@@ -0,0 +1,15 @@
### Rotate backup log file
# location of log file (-l parameter of script file)
/path/to/pilerbackup.log {
# rotate log file weekly -- you could also use 'daily', 'monthly' or
# specify a size using 'size 100k', for example
weekly
# keep 4 weeks of old logs (4 files) and delete older ones
rotate 4
# compress old log files using gzip (default)
compress
}
@@ -0,0 +1,13 @@
# Location of your script's log file, -l parameter
LogFile = /path/to/your/pilerbackup.log
# Format of logrotate archives for your script. Example assumes compression and
# extension preservation
Archive = /path/to/your/pilerbackup.log.?.gz
# Apply the correct date/time filtering to match the format of the script's log
# We are using a custom pl script in /etc/logwatch/scripts/shared/ You don't
# need to change this unless you have altered the 'stamp' function in the backup
# script in which case you will want to update the regex in the custom pl script
# below
*sqFullStampAnywhere
@@ -0,0 +1,9 @@
# Name of the logfile group without any extension
LogFile = pilerbackup
# Heading displayed on Logwatch's report for this service
Title = "Archived mail backup (piler)"
# Override the detail level for this service
# Remember the levels are: 0, 1-4, 5, 6+
# Detail = 0
+112
View File
@@ -0,0 +1,112 @@
#!/usr/bin/perl
#############################################################################
# $Id$
#############################################################################
# Log: Piler exported EML backup (pilerbackup, Logwatch group 'pilerbackup')
# Revision 1.1 2019/07/20
# Written by Asif Bacchus
#############################################################################
use strict;
### Get Logwatch detail level (default to 0)
my $detailLevel = $ENV{'LOGWATCH_DETAIL_LEVEL'} || 0;
### Declare variables
my $summaryErr;
my $summaryWarn;
my $summarySuccess;
my %reportHash = ();
my $key;
### Minimal detail level: provide summary data only
if ($detailLevel == 0) {
### process logfile and summarize message types
while (defined(my $ThisLine = <STDIN>)) {
if ($ThisLine =~ /\-- \[ERROR\] /) {
$summaryErr++;
}
elsif ($ThisLine =~ /\-- \[WARNING\] /) {
$summaryWarn++;
}
elsif ($ThisLine =~ /All processes completed/) {
$summarySuccess++;
}
}
### fill hash table with headings and summary counts
if ($summarySuccess > 0) {
$reportHash{"All processes successfully completed"} = $summarySuccess;
}
if ($summaryWarn > 0) {
$reportHash{"Warnings issued"} = $summaryWarn;
}
if ($summaryErr > 0) {
$reportHash{"Errors encountered"} = $summaryErr;
}
### print hash table
foreach $key (sort keys %reportHash) {
print "$key: $reportHash{$key}\n";
}
}
### Levels 1-4 provide the actual error, warning and success messages instead
### of a summary count
elsif ($detailLevel >= 1 && $detailLevel <= 4) {
while (defined(my $ThisLine = <STDIN>)) {
if ($ThisLine =~ /\-- \[ERROR\] /) {
print $ThisLine;
}
elsif ($ThisLine =~ /\-- \[WARNING\] /) {
print $ThisLine;
}
elsif ($ThisLine =~ /\-- \[SUCCESS\] /) {
print $ThisLine;
}
}
}
### Level 5 is similiar to levels 1-4 except it also reports informational
### messages such as the location of script created files, variable checks,
### etc. This is useful when verifying the script's operation.
elsif ($detailLevel == 5) {
while (defined(my $ThisLine = <STDIN>)) {
if ($ThisLine =~ /\-- \[ERROR\] /) {
print $ThisLine;
}
elsif ($ThisLine =~ /\-- \[WARNING\] /) {
print $ThisLine;
}
elsif ($ThisLine =~ /\-- \[SUCCESS\] /) {
print $ThisLine;
}
elsif ($ThisLine =~ /\-- \[INFO\] /) {
print $ThisLine;
}
}
}
### Any level above 5 will echo the entire log including the debugging notes
### within the script meant for troubleshooting. Using this level of detail
### should only be done if you cannot view the actual log file directly for
### whatever reason. The actual log file is colour-coded for easier debugging.
elsif ($detailLevel > 5) {
while (defined(my $ThisLine = <STDIN>)) {
print $ThisLine;
}
}
### Exit gracefully
exit (0);
# vi: shiftwidth=3 tabstop=3 et
# Local Variables:
# mode: perl
# perl-indent-level: 3
# indent-tabs-mode: nil
# End:
@@ -0,0 +1,37 @@
#!/usr/bin/perl
##########################################################################
# $Id$
##########################################################################
###############################################################################
## Filter dates in full-date-time international format, surrounded by square
## brackets located anywhere on a given line
## Format: '[%Y-%m-%d %H:%M:%S]'
###############################################################################
use Logwatch ':dates';
my $Debug = $ENV{'LOGWATCH_DEBUG'} || 0;
### Specify the format of the date/time stamp itself
$SearchDate = TimeFilter('%Y-%m-%d %H:%M:%S');
if ( $Debug > 5 ) {
print STDERR "DEBUG: Inside FullDateTime...\n";
print STDERR "DEBUG: Looking For: " . $SearchDate . "\n";
}
while (defined($ThisLine = <STDIN>)) {
### specify the regex that defines how to find 'SearchDate'
if ($ThisLine =~ m/\[$SearchDate\] /o) {
print $ThisLine;
}
}
# vi: shiftwidth=3 syntax=perl tabstop=3 et
# Local Variables:
# mode: perl
# perl-indent-level: 3
# indent-tabs-mode: nil
# End:
+26
View File
@@ -0,0 +1,26 @@
<!DOCTYPE html>
<html>
<head>
<meta charset="utf-8" />
<meta http-equiv="X-UA-Compatible" content="IE=edge">
<title>503 - Unavailable: Backup in progress</title>
<meta name="viewport" content="width=device-width, initial-scale=1">
<style>
body {
margin: 0 auto;
padding: 0;
width: 85%;
font-family: 'Segoe UI', Tahoma, Geneva, Verdana, sans-serif
}
</style>
</head>
<body>
<h1>Bad timing!</h1>
<p>Seems you're trying to access me during my daily backup window. Don't worry though, I should be up and running again very soon.</p>
<p>My average backup window duration is pretty short and I'm quite busy during that time copying your super-important stuff to my secure hiding place so they stay safe in case anything ever happens to me!</p>
<h3><em>I'm really sorry for the delay. Please try me again soon!</em></h3>
</body>
</html>
@@ -32,6 +32,12 @@ borgConnectRepo="jdoe123@borg.server.net:pilerBackup/"
# recommended for security
borgRepoPassphrase="p@ssW0rd"
# keyfile to access repo
# FULL PATH where the associated keyfile for your repo is located -- relevant
# only if your repo requires a keyfile (i.e. 'keyfile' vs 'repokey') and if you
# are not using the default keyfile location
borgKeyfileLocation="/var/borgbackup/.config/borg/keys/server_address__repo_name"
# OPTIONAL: path to text file containing a list (one per line) of files/
# directories to include in your backup along with exported data from piler
# see repo wiki for more details
+686
View File
@@ -0,0 +1,686 @@
#!/bin/sh
#######
### Backup exported email from piler (bare-metal)
#######
### set environment variables
if [ -z "$TERM" ]; then
export TERM=dumb
fi
export PATH=$PATH:/usr/local/bin
### text formatting presents
if command -v tput >/dev/null; then
bold=$(tput bold)
cyan=$(tput setaf 6)
err=$(tput bold)$(tput setaf 1)
magenta=$(tput setaf 5)
norm=$(tput sgr0)
ok=$(tput setaf 2)
warn=$(tput bold)$(tput setaf 3)
width=$(tput cols)
yellow=$(tput setaf 3)
else
bold=""
cyan=""
err=""
magenta=""
norm=""
ok=""
warn=""
width=80
yellow=""
fi
### trap
trap trapExit 1 2 3 6
### functions
# bad configuration value passed in details file
badDetails() {
if [ "$1" = "empty" ]; then
exitError 130 "details:${2} cannot be NULL (undefined)"
elif [ "$1" = "dne" ]; then
exitError 131 "details:${2} file or directory does not exist."
fi
}
# bad parameter passed to script
badParam() {
if [ "$1" = "dne" ]; then
printf "\n%sError: '%s %s'\n" "$err" "$2" "$3"
printf "file or directory does not exist.%s\n\n" "$norm"
exit 1
elif [ "$1" = "empty" ]; then
printf "\n%sError: '%s' cannot have a NULL (empty) value.\n" "$err" "$2"
printf "%sPlease use '--help' for assistance%s\n\n" "$cyan" "$norm"
exit 1
elif [ "$1" = "svc" ]; then
printf "\n%sError: '%s %s': Service does not exist!${norm}\n\n" \
"$err" "$2" "$3"
exit 1
elif [ "$1" = "user" ]; then
printf "\n%sError: '%s %s': User does not exist!${norm}\n\n" \
"$err" "$2" "$3"
exit 1
fi
}
# cleanup
cleanup() {
# cleanup 503 if copied
if [ "$err503Copied" -eq 1 ]; then
if ! rm -f "$webroot/$err503File" 2>>"$logFile"; then
# shellcheck disable=SC2129
printf "%s[%s] -- [WARNING] Could not remove 503 error page." \
"$warn" "$(stamp)" >>"$logFile"
printf " Web interface will not function until this file is " \
>>"$logFile"
printf "removed --%s\n" "$norm" >>"$logFile"
warnCount=$((warnCount + 1))
else
printf "%s[%s] -- [INFO] 503 error page removed --%s\n" \
"$cyan" "$(stamp)" "$norm" >>"$logFile"
fi
fi
# remove EML temporary directory
if ! rm -rf "$exportDir" 2>>"$logFile"; then
# shellcheck disable=SC2129
printf "%s[%s] -- [WARNING] Could not remove EML export tmp dir:" \
"$warn" "$(stamp)" >>"$logFile"
printf "\n%s\n" "$exportDir" >>"$logFile"
printf "please remove this directory manually! --%s\n" "$norm" \
>>"$logFile"
warnCount=$((warnCount + 1))
else
printf "%s[%s] -- [INFO] EML temp directory removed --%s\n" \
"$cyan" "$(stamp)" "$norm" >>"$logFile"
fi
}
# call cleanup and then exit with error report
exitError() {
printf "%s[%s] -- [ERROR] %s: %s --%s\n" \
"$err" "$(stamp)" "$1" "$2" "$norm" >>"$logFile"
cleanup
# note script completion with error
printf "%s[%s] --- %s execution completed with error ---%s\n" \
"$err" "$(stamp)" "$scriptName" "$norm" >>"$logFile"
exit "$1"
}
# display script help information
scriptHelp() {
newline
printf "%sUsage: %s [parameters]%s\n\n" "$bold" "$scriptName" "$norm"
textBlock "There are no mandatory parameters. If a parameter is not supplied, its default value will be used. In the case of a switch parameter, it will remain DEACTIVATED by default."
newline
textBlock "Parameters are listed then followed by a description of their effect on the following line. Finally, if a default value exists, it will be listed on the next line in (parentheses)."
newline
textBlock "${magenta}--- script related ---${norm}"
newline
textBlockSwitch "-c | --config | --details"
textBlock "Path to the configuration key/value pair file for this script."
textBlockDefaults "(scriptPath/scriptName.details)"
newline
textBlockSwitch "-h | -? | --help"
textBlock "This help screen."
newline
textBlockSwitch "-l | --log"
textBlock "Path to write log file."
textBlockDefaults "(scriptPath/scriptName.log)"
newline
textBlockSwitch "[SWITCH] -v | --verbose"
textBlock "Log borg output with increased verbosity (list all files). Careful! Your log file can get very large very quickly!"
textBlockDefaults "(normal output, option is OFF)"
newline
textBlock "${magenta}--- export options ---${norm}"
newline
textBlockSwitch "[SWITCH] -a | --all"
textBlock "Export ALL email saved by piler. This is generally only useful for an initial backup. Setting this switch will override any specified start/end dates."
textBlockDefaults "(use start/end dates, option is OFF)"
newline
textBlockSwitch "--start"
textBlock "Export email starting from this date (inclusive). Date MUST be provided in YYYY.MM.DD format."
textBlockDefaults "(yesterday's date)"
newline
textBlockSwitch "--end"
textBlock "Export email ending at this date (inclusive). Date MUST be provided in YYYY.MM.DD format."
textBlockDefaults "(yesterday's date)"
newline
textBlock "${magenta}--- 503 functionality ---${norm}"
newline
textBlockSwitch "[SWITCH] -5 | --use-503"
textBlock "Copy an 'error 503' page/indicator file to your webroot for your webserver to find. Specifying this option will enable other 503 options."
textBlockDefaults "(do NOT copy, option is OFF)"
newline
textBlockSwitch "--503-path"
textBlock "Path to the file you want copied to your webroot as the 'error 503' page."
textBlockDefaults "(scriptPath/503_backup.html)"
newline
textBlockSwitch "-w | --webroot"
textBlock "Path to where the 'error 503' file should be copied."
textBlockDefaults "(/usr/share/nginx/html/)"
newline
textBlock "More details and examples of script usage can be found in the repo wiki at ${yellow}https://git.asifbacchus.app/asif/pilerBackup/wiki${norm}"
newline
}
# generate dynamic timestamps
stamp() {
(date +%F" "%T)
}
textBlock() {
printf "%s\n" "$1" | fold -w "$width" -s
}
textBlockDefaults() {
printf "%s%s%s\n" "$yellow" "$1" "$norm"
}
textBlockSwitch() {
printf "%s%s%s\n" "$cyan" "$1" "$norm"
}
newline() {
printf "\n"
}
# same as exitError but for signal captures
trapExit() {
printf "%s[%s] -- [ERROR] 99: Caught signal --%s\n" \
"$err" "$(stamp)" "$norm" >>"$logFile"
cleanup
# note script completion with error
printf "%s[%s] --- %s execution was terminated via signal ---%s\n" \
"$err" "$(stamp)" "$scriptName" "$norm" >>"$logFile"
exit 99
}
### end of functions
### default variable values
## script related
# store logfile in the same directory as this script file using the same file
# name as the script but with the extension '.log'
scriptPath="$(CDPATH='' cd -- "$(dirname -- "$0")" && pwd -P)"
scriptName="$(basename "$0")"
logFile="$scriptPath/${scriptName%.*}.log"
warnCount=0
configDetails="$scriptPath/${scriptName%.*}.details"
err503Copied=0
includeXtra=0
exclusions=0
# borg output verbosity -- normal
borgCreateParams='--stats'
borgPruneParams='--list'
# 503 related
use503=0
err503Path="$scriptPath/503_backup.html"
err503File="${err503Path##*/}"
webroot="/usr/share/nginx/html"
# piler-export related
exportAll=0
exportStart=$(date -d 'yesterday' +'%Y.%m.%d')
exportEnd=$(date -d 'yesterday' +'%Y.%m.%d')
pilerUser='piler'
### process startup parameters
while [ $# -gt 0 ]; do
case "$1" in
-h | -\? | --help)
# display help
scriptHelp
exit 0
;;
-l | --log)
# set log file location
if [ -n "$2" ]; then
logFile="${2%/}"
shift
else
badParam empty "$@"
fi
;;
-v | --verbose)
# set verbose logging from borg
borgCreateParams='--list --stats'
borgPruneParams='--list'
;;
-c | --config | --details)
# location of config details file
if [ -n "$2" ]; then
if [ -f "$2" ]; then
configDetails="${2%/}"
shift
else
badParam dne "$@"
fi
else
badParam empty "$@"
fi
;;
-5 | --use-503)
# enable copying 503 error page to webroot
use503=1
;;
--503-path)
# FULL path to 503 file
if [ -n "$2" ]; then
if [ -f "$2" ]; then
err503Path="${2%/}"
err503File="${2##*/}"
shift
else
badParam dne "$@"
fi
else
badParam empty "$@"
fi
;;
-w | --webroot)
# path to webroot (copy 503)
if [ -n "$2" ]; then
if [ -d "$2" ]; then
webroot="${2%/}"
shift
else
badParam dne "$@"
fi
else
badParam empty "$@"
fi
;;
-a | --all)
# export ALL email stored in piler
exportAll=1
;;
--start)
# export starting at this date
if [ -n "$2" ]; then
exportStart="$2"
shift
else
badParam empty "$@"
fi
;;
--end)
# export ending at this date
if [ -n "$2" ]; then
exportEnd="$2"
shift
else
badParam empty "$@"
fi
;;
--user)
# specify piler user account name
if [ -n "$2" ]; then
if id ${pilerUser} >/dev/null 2>&1; then
pilerUser="$2"
shift
else
badParam user "$@"
fi
else
badParam empty "$@"
fi
;;
*)
printf "%s\nUnknown option: %s\n" "$err" "$1"
printf "%sUse '--help' for valid options.%s\n\n" "$cyan" "$norm"
exit 1
;;
esac
shift
done
### check pre-requisites and default values
# check if running as root, otherwise exit
if [ "$(id -u)" -ne 0 ]; then
printf "\n%sERROR: script MUST be run as ROOT%s\n\n" "$err" "$norm"
exit 2
fi
# find pilerexport, otherwise this is all pointless
pilerExport=$(command -v pilerexport)
if [ ! "$pilerExport" ]; then
printf "\n%sERROR: cannot find 'pilerexport'%s\n\n" "$err" "$norm"
exit 4
fi
# does the details file exist?
if [ ! -f "$configDetails" ]; then
badParam dne "(--details default)" "$configDetails"
fi
# is borg installed?
if ! command -v borg >/dev/null; then
printf "\n%sERROR: BORG is not installed on this system!%s\n\n" "$err" "$norm"
exit 3
fi
# if 503 functionality is enabled, do 503 related files exist?
if [ "$use503" -eq 1 ]; then
if [ ! -f "$err503Path" ]; then
badParam dne "(--503-path default)" "$err503Path"
elif [ ! -d "$webroot" ]; then
badParam dne "(--webroot default)" "$webroot"
fi
fi
### start logging
printf "%s[%s] --- Start %s execution ---%s\n" \
"$magenta" "$(stamp)" "$scriptName" "$norm" >>"$logFile"
printf "%s[%s] -- [INFO] Log located at %s%s%s --%s\n" \
"$cyan" "$(stamp)" "$yellow" "$logFile" "$cyan" "$norm" >>"$logFile"
### 503 functionality
if [ "$use503" -eq 1 ]; then
printf "%s[%s] -- [INFO] Copying 503 error page to " \
"$cyan" "$(stamp)" >>"$logFile"
printf "webroot -- %s\n" "$norm" >>"$logFile"
if ! \cp --force "${err503Path}" "${webroot}/${err503File}" 2>>"$logFile"; then
printf "%s[%s] -- [WARNING] Failed to copy 503 error page. " \
"$warn" "$(stamp)" >>"$logFile"
printf "Web users will NOT be notified --%s\n" "$norm" >>"$logFile"
warnCount=$((warnCount + 1))
else
printf "%s[%s] -- [SUCCESS] 503 error page copied --%s\n" \
"$ok" "$(stamp)" "$norm" >>"$logFile"
# set cleanup flag
err503Copied=1
fi
fi
### read details file to get variables needed to dump sql and run borg
# check if config details file was provided as a relative or absolute path
case "${configDetails}" in
/*)
# absolute path, no need to rewrite variable
# shellcheck source=pilerbackup.details
. "${configDetails}"
;;
*)
# relative path, prepend './' to create absolute path
# shellcheck source=pilerbackup.details
. "./${configDetails}"
;;
esac
printf "%s[%s] -- [INFO] %s%s%s imported --%s\n" \
"$cyan" "$(stamp)" "$yellow" "$configDetails" "$cyan" "$norm" >>"$logFile"
## create tmp directory and change to it for export operations
# create temporary directory to dump exported email from piler
tmpdir="/tmp/emailbackup/$(date +%F_%T)"
if ! exportDir=$(mkdir -p "${tmpdir}" 2>>"$logFile"); then
exitError 111 "Could not create temporary directory for exported EML files"
fi
# grant pilerUser permission to write to temporary directory
if # shellcheck disable=SC2086
! (chown root:${pilerUser} "$exportDir" && chmod 770 "$exportDir")
then
exitError 112 "Could not set permissions on temporary directory"
fi
if ! cd "$exportDir"; then
exitError 113 "Unable to change to temporary export directory"
fi
# shellcheck disable=SC2129
printf "%s[%s] -- [INFO] EML files will be temporarily stored in:" \
"$cyan" "$(stamp)" >>"$logFile"
printf "\n\t%s%s/%s --%s\n" "$yellow" "$exportDir" "$cyan" "$norm" >>"$logFile"
## export EML from piler
printf "%s[%s] -- [INFO] Exporting EML files from piler --%s\n" \
"$cyan" "$(stamp)" "$norm" >>"$logFile"
if [ "$exportAll" -eq 1 ]; then
if ! ${pilerExport} -A 2>>"$logFile"; then
exitError 115 "There was a problem while exporting EML from piler"
fi
printf "%s[%s] -- [SUCCESS] Exported EML files from piler --%s\n" \
"$ok" "$(stamp)" "$norm" >>"$logFile"
else
if ! ${pilerExport} -a "$exportStart" -b "$exportEnd" 2>>"$logFile"; then
exitError 115 "There was a problem while exporting EML from piler"
fi
printf "%s[%s] -- [SUCCESS] Exported EML files from piler --%s\n" \
"$ok" "$(stamp)" "$norm" >>"$logFile"
fi
### pre-backup tasks completed -- move to borg tasks
printf "%s[%s] -- [SUCCESS] Pre-backup tasks completed --%s\n" \
"$ok" "$(stamp)" "$norm" >>"$logFile"
### Run borg variable checks
printf "%s[%s] -- [INFO] Verifying supplied borg details --%s\n" \
"$cyan" "$(stamp)" "$norm" >>"$logFile"
## verify borg base directory
if [ -z "${borgBaseDir}" ]; then
badDetails empty 'borgBaseDir'
elif [ ! -d "${borgBaseDir}" ]; then
badDetails dne 'borgBaseDir'
fi
printf "%sdetails:borgBaseDir%s -- %s[OK]%s\n" \
"$magenta" "$norm" "$ok" "$norm" >>"$logFile"
export BORG_BASE_DIR="${borgBaseDir%/}"
## check path to SSH keyfile
if [ -z "${borgSSHKey}" ]; then
badDetails empty 'borgSSHKey'
elif [ ! -f "${borgSSHKey}" ]; then
badDetails dne 'borgSSHKey'
fi
printf "%sdetails:borgSSHKey%s -- %s[OK]%s\n" \
"$magenta" "$norm" "$ok" "$norm" >>"$logFile"
export BORG_RSH="ssh -i ${borgSSHKey}"
## check borg repo connect string
if [ -z "${borgConnectRepo}" ]; then
badDetails empty 'borgConnectRepo'
fi
printf "%sdetails:borgConnectRepo%s -- %s[OK]%s\n" \
"$magenta" "$norm" "$ok" "$norm" >>"$logFile"
export BORG_REPO="${borgConnectRepo}"
## check borg repo password
if [ -n "${borgRepoPassphrase}" ]; then
printf "%sdetails:borgRepoPassphrase%s -- %s[OK]%s\n" \
"$magenta" "$norm" "$ok" "$norm" >>"$logFile"
export BORG_PASSPHRASE="${borgRepoPassphrase}"
else
# if passwd is blank intentionally, this is insecure
printf "%s-- [WARNING] Using a borg repo without a password is an " \
"$warn" >>"$logFile"
printf "insecure configuration --%s\n" "$norm" >>"$logFile"
warnCount=$((warnCount + 1))
# if this was an accident, we need to provide a bogus passwd so borg fails
# otherwise it will sit forever just waiting for input
export BORG_PASSPHRASE="DummyPasswordSoBorgFails"
fi
## check borg repository keyfile location
if [ -z "${borgKeyfileLocation}" ]; then
printf "%sdetails:borgKeyfileLocation %s-- %s[DEFAULT]%s\n" "$magenta" "$norm" "$ok" "$norm" >>"$logFile"
else
# check if keyfile location exists
if [ ! -f "${borgKeyfileLocation}" ]; then
badDetails dne 'borgKeyfileLocation'
fi
printf "%sdetails:borgKeyfileLocation %s-- %s[OK]%s\n" "$magenta" "$norm" "$ok" "$norm" >>"$logFile"
export BORG_KEY_FILE="${borgKeyfileLocation}"
fi
## export borg remote path, if specified
if [ -n "${borgRemote}" ]; then export BORG_REMOTE_PATH="${borgRemote}"; fi
## read additional files
if [ -n "${borgXtraListPath}" ]; then
# check if file actually exists
if [ ! -f "${borgXtraListPath}" ]; then
badDetails dne 'borgXtraListPath'
fi
# read file contents into concatenated list for echo to cmdline
while read -r xtraItem; do
if [ -z "${xtraList}" ]; then
xtraList="${xtraItem}"
else
xtraList="${xtraList} ${xtraItem}"
fi
done <<EOF
$(sed -e '/^\s*#.*$/d' -e '/^\s*$/d' "${borgXtraListPath}")
EOF
printf "%sdetails:borgXtraListPath%s -- %s[OK]%s\n" \
"$magenta" "$norm" "$ok" "$norm" >>"$logFile"
includeXtra=1
fi
## check if exclusion list file is specified
if [ -n "${borgExcludeListPath}" ]; then
# check if the file actually exists
if [ ! -f "${borgExcludeListPath}" ]; then
badDetails dne 'borgExcludeListPath'
fi
exclusions=1
fi
### create borg temp dir:
## python requires a writable temporary directory when unpacking borg and
## executing commands. This defaults to /tmp but many systems mount /tmp with
## the 'noexec' option for security. Thus, we will use/create a 'tmp' folder
## within the BORG_BASE_DIR and instruct python to use that instead of /tmp
# check if BORG_BASE_DIR/tmp exists, if not, create it
if [ ! -d "${borgBaseDir}/tmp" ]; then
if ! mkdir "${borgBaseDir}/tmp"; then
exitError 132 "Unable to create borg ${borgBaseDir}/tmp directory"
else
printf "%s[%s] -- [INFO] Created %s%s/tmp " \
"$cyan" "$(stamp)" "$yellow" "${borgBaseDir}" >>"$logFile"
printf "%s--%s\n" "$cyan" "$norm" >>"$logFile"
fi
fi
export TMPDIR="${borgBaseDir}/tmp"
### execute borg depending on whether extra files and/or exclusions are defined
## construct the proper borg commandline
# base command
if [ "$exclusions" -eq 0 ]; then
borgCMD="borg --show-rc create ${borgCreateParams} \
::$(date +%Y-%m-%d_%H%M%S) \
${exportDir}"
elif [ "$exclusions" -eq 1 ]; then
borgCMD="borg --show-rc create ${borgCreateParams} \
--exclude-from ${borgExcludeListPath} \
::$(date +%Y-%m-%d_%H%M%S) \
${exportDir}"
fi
# add extra locations if defined
if [ "$includeXtra" -eq 1 ]; then
borgCMD="${borgCMD} ${xtraList}"
fi
# execute borg
printf "%s[%s] -- [INFO] Executing borg backup operation --%s\n" \
"$cyan" "$(stamp)" "$norm" >>"$logFile"
${borgCMD} 2>>"$logFile"
borgResult="$?"
## check borg exit status
if [ "$borgResult" -eq 0 ]; then
printf "%s[%s] -- [SUCCESS] Borg backup completed --%s\n" \
"$ok" "$(stamp)" "$norm" >>"$logFile"
elif [ "$borgResult" -eq 1 ]; then
printf "%s[%s] -- [WARNING] Borg completed with warnings. " \
"$warn" "$(stamp)" >>"$logFile"
printf "Review this logfile for details --%s\n" "$norm" >>"$logFile"
warnCount=$((warnCount + 1))
elif [ "$borgResult" -ge 2 ]; then
err_1="Borg exited with a critical error. Please review this log file"
err_2="for details."
exitError 138 "$err_1 $err_2"
else
printf "%s[%s] -- [WARNING] Borg exited with unknown return code. " \
"$warn" "$(stamp)" >>"$logFile"
printf "Review this logfile for details --%s\n" "$norm" >>"$logFile"
warnCount=$((warnCount + 1))
fi
### execute borg prune if parameters are provided, otherwise skip with a warning
if [ -n "${borgPruneSettings}" ]; then
printf "%s[%s] -- [INFO] Executing borg prune operation --%s\n" \
"$cyan" "$(stamp)" "$norm" >>"$logFile"
# shellcheck disable=SC2086
borg prune --show-rc -v ${borgPruneParams} ${borgPruneSettings} \
2>>"$logFile"
borgPruneResult="$?"
else
printf "%s[%s] -- [WARNING] No prune parameters provided. " \
"$warn" "$(stamp)" >>"$logFile"
printf "Your archive will continue growing with each backup --%s\n" \
"$norm" >>"$logFile"
warnCount=$((warnCount + 1))
fi
## report on prune operation if executed
if [ -n "${borgPruneResult}" ]; then
if [ "${borgPruneResult}" -eq 0 ]; then
printf "%s[%s] -- [SUCCESS] Borg prune completed --%s\n" \
"$ok" "$(stamp)" "$norm" >>"$logFile"
elif [ "$borgPruneResult" -eq 1 ]; then
printf "%s[%s] -- [WARNING] Borg prune completed with warnings. " \
"$warn" "$(stamp)" >>"$logFile"
printf "Review this logfile for details --%s\n" "$norm" >>"$logFile"
warnCount=$((warnCount + 1))
elif [ "$borgPruneResult" -ge 2 ]; then
err_1="Borg prune exited with a critical error. Please review this"
err_2="log file for details."
exitError 139 "$err_1 $err_2"
else
printf "%s[%s] -- [WARNING] Borg prune exited with an unknown " \
"$warn" "$(stamp)" >>"$logFile"
printf "return code. Review this logfile for details --%s\n" \
"$norm" >>"$logFile"
warnCount=$((warnCount + 1))
fi
fi
### all processes successfully completed, cleanup and exit gracefully
# note successful completion of borg commands
printf "%s[%s] -- [SUCCESS] Backup operations completed --%s\n" \
"$ok" "$(stamp)" "$norm" >>"$logFile"
# cleanup
cleanup
# note complete success, tally warnings and exit
printf "%s[%s] -- [SUCCESS] All processes completed --%s\n" \
"$ok" "$(stamp)" "$norm" >>"$logFile"
printf "%s[%s] --- %s execution completed ---%s\n" \
"$magenta" "$(stamp)" "$scriptName" "$norm" >>"$logFile"
if [ "$warnCount" -gt 0 ]; then
printf "%s%s warnings issued!%s\n" "$warn" "${warnCount}" "$norm" >>"$logFile"
else
printf "%s0 warnings issued.%s\n" "$ok" "$norm" >>"$logFile"
fi
exit 0
### error codes
# 1: parameter error
# 2: not run as root
# 3: borg not installed
# 4: cannot find pilerexport executable
# 99: TERM signal trapped
# 111: could not create tmp dir for EML dump from piler-export
# 112: could not set permissions on tmp dir
# 113: could not change to tmp dir to start export operation
# 115: problem during piler-export operation
# 130: null configuration variable in details file
# 131: invalid configuration variable in details file
# 138: borg exited with a critical error
# 139: borg prune exited with a critical error
@@ -31,7 +31,9 @@
# root user files
/root/.bashrc
/root/.ssh/
/root/scripts/
# scripts
/scripts/
### piler configuration files and KEY!
-627
View File
@@ -1,627 +0,0 @@
#!/bin/sh
#######
### Backup exported email from piler (bare-metal)
#######
### text formatting presents
norm=$(tput sgr0)
err=$(tput bold)$(tput setaf 1)
warn=$(tput bold)$(tput setaf 3)
ok=$(tput setaf 2)
yellow=$(tput setaf 3)
cyan=$(tput setaf 6)
magenta=$(tput setaf 5)
### trap
trap trapExit 1 2 3 6
### functions
# bad configuration value passed in details file
badDetails () {
if [ "$1" = "empty" ]; then
exitError 130 "details:${2} cannot be NULL (undefined)"
elif [ "$1" = "dne" ]; then
exitError 131 "details:${2} file or directory does not exist."
fi
}
# bad parameter passed to script
badParam () {
if [ "$1" = "dne" ]; then
printf "${err}\nError: '%s %s'\n" "$2" "$3"
printf "file or directory does not exist.${norm}\n\n"
exit 1
elif [ "$1" = "empty" ]; then
printf "${err}\nError: '%s' cannot have a NULL (empty) value.\n" "$2"
printf "${cyan}Please use '--help' for assistance${norm}\n\n"
exit 1
elif [ "$1" = "svc" ]; then
printf "${err}\nError: '%s %s': Service does not exist!${norm}\n\n" \
"$2" "$3"
exit 1
fi
}
# cleanup
cleanup () {
# cleanup 503 if copied
if [ "$err503Copied" -eq 1 ]; then
if ! rm -f "$webroot/$err503File" 2>>"$logFile"; then
printf "${warn}[%s] -- [WARNING] Could not remove 503 error page." \
"$(stamp)" >> "$logFile"
printf " Web interface will not function until this file is " \
>> "$logFile"
printf "removed --${norm}\n" >> "$logFile"
warnCount=$((warnCount+1))
else
printf "${cyan}[%s] -- [INFO] 503 error page removed --${norm}\n" \
"$(stamp)" >> "$logFile"
fi
fi
# remove EML temporary directory
if ! rm -f "$exportDir" 2>>"$logFile"; then
printf "${warn}[%s] -- [WARNING] Could not remove EML export tmp dir:" \
"$(stamp)" >> "$logFile"
printf "\n%s\n" "$exportDir" >> "$logFile"
printf "please remove this directory manually! --${norm}\n" \
>> "$logFile"
warnCount=$((warnCount+1))
else
printf "${cyan}[%s] -- [INFO] EML temp directory removed --${norm}\n" \
"$(stamp)" >> "$logFile"
fi
}
# call cleanup and then exit with error report
exitError () {
printf "${err}[%s] -- [ERROR] %s: %s --${norm}\n" \
"$(stamp)" "$1" "$2" >> "$logFile"
cleanup
# note script completion with error
printf "${err}[%s] --- %s execution completed with error ---${norm}\n" \
"$(stamp)" "$scriptName" >> "$logFile"
exit "$1"
}
# display script help information
scriptHelp () {
printf "\n${cyan}%s\n" "--------------------------------------------------------------------------------"
printf "Usage:${norm} %s [parameters]\n\n" "$scriptName"
printf "${cyan}Parameters ${yellow}(default value):${norm}\n"
printf "There are NO mandatory parameters. If a parameter is not "
printf "supplied, it's\n"
printf "default value will be used. In the case of a switch parameter, "
printf "it will\n"
printf "remain deactivated if not specified.\n\n"
printf "${magenta}script related\n"
printf "${cyan}-c, --config, --details${norm}\n"
printf "Path to the configuration key/value pair file for this script.\n"
printf "${yellow}(scriptPath/scriptName.details)${norm}\n\n"
printf "${cyan}-h, -?, --help${norm}\n"
printf "This help screen.\n\n"
printf "${cyan}-l, --log${norm}\n"
printf "Path to write log file.\n"
printf "${yellow}(scriptPath/scriptName.log)${norm}\n\n"
printf "${cyan}-v, --verbose${norm}\n"
printf "Log borg output with increased verbosity (list all files). "
printf "Careful! Your\n"
printf "log file can get large very quickly! This is a switch value, "
printf "specifying it\n"
printf "will turn the option ON.\n"
printf "${yellow}(normal output, option is OFF)${norm}\n\n"
printf "${magenta}export options\n"
printf "${cyan}-a, --all${norm}\n"
printf "Export ALL email saved by piler. This is generally only useful "
printf "for an initial\n"
printf "backup. This is a switch parameter and WILL override specified "
printf "start/end dates.\n"
printf "${yellow}(option is OFF, use start/end dates)${norm}\n\n"
printf "${cyan}--start${norm}\n"
printf "Export email starting from this date (inclusive)\n"
printf "Date MUST be provided in the format 'YYYY.MM.DD'\n"
printf "${yellow}(Yesterday's date)${norm}\n\n"
printf "${cyan}--end${norm}\n"
printf "Export email ending at this date (inclusive)\n"
printf "Date MUST be provided in the format 'YYYY.MM.DD'\n"
printf "${yellow}(Yesterday's date)${norm}\n\n"
printf "${magenta}503 functionality\n"
printf "${cyan}-5, --use-503${norm}\n"
printf "Copy an 'error 503' page to your webroot for your webserver to "
printf "find. This is a\n"
printf "switch value, specifying it will instruct the script to copy the "
printf "error page.\n"
printf "${yellow}(do NOT copy, option is OFF)${norm}\n\n"
printf "${cyan}--503-path${norm}\n"
printf "Path to the file you want copied to your webroot as the 'error "
printf "503' page during\n"
printf "backup operations.\n"
printf "${yellow}(scriptPath/503_backup.html)${norm}\n\n"
printf "${cyan}-w, --webroot${norm}\n"
printf "Path to where the 'error 503' file should be copied during "
printf "backups.\n"
printf "${yellow}(/usr/share/nginx/html)${norm}\n\n"
printf "More details and examples of script usage can be found in the "
printf "repo wiki at\n"
printf "${yellow}https://git.asifbacchus.app/asif/pilerBackup/wiki${norm}\n"
printf "${cyan}%s${norm}\n\n" "--------------------------------------------------------------------------------"
}
# generate dynamic timestamps
stamp () {
(date +%F" "%T)
}
# same as exitError but for signal captures
trapExit () {
printf "${err}[%s] -- [ERROR] 99: Caught signal --${norm}\n" \
"$(stamp)" >> "$logFile"
cleanup
# note script completion with error
printf "${err}[%s] --- %s execution was terminated via signal ---${norm}\n" \
"$(stamp)" "$scriptName" >> "$logFile"
exit 99
}
### end of functions
### default variable values
## script related
# store logfile in the same directory as this script file using the same file
# name as the script but with the extension '.log'
scriptPath="$( CDPATH='' cd -- "$( dirname -- "$0" )" && pwd -P )"
scriptName="$( basename "$0" )"
logFile="$scriptPath/${scriptName%.*}.log"
warnCount=0
configDetails="$scriptPath/${scriptName%.*}.details"
err503Copied=0
includeXtra=0
exclusions=0
# borg output verbosity -- normal
borgCreateParams='--stats'
borgPruneParams='--list'
# 503 related
use503=0
err503Path="$scriptPath/503_backup.html"
err503File="${err503Path##*/}"
webroot="/usr/share/nginx/html"
# piler-export related
exportAll=0
exportStart=$( date -d 'yesterday' +'%Y.%m.%d' )
exportEnd=$( date -d 'yesterday' +'%Y.%m.%d' )
### process startup parameters
while [ $# -gt 0 ]; do
case "$1" in
-h|-\?|--help)
# display help
scriptHelp
exit 0
;;
-l|--log)
# set log file location
if [ -n "$2" ]; then
logFile="${2%/}"
shift
else
badParam empty "$@"
fi
;;
-v|--verbose)
# set verbose logging from borg
borgCreateParams='--list --stats'
borgPruneParams='--list'
;;
-c|--config|--details)
# location of config details file
if [ -n "$2" ]; then
if [ -f "$2" ]; then
configDetails="${2%/}"
shift
else
badParam dne "$@"
fi
else
badParam empty "$@"
fi
;;
-5|--use-503)
# enable copying 503 error page to webroot
use503=1
;;
--503-path)
# FULL path to 503 file
if [ -n "$2" ]; then
if [ -f "$2" ]; then
err503Path="${2%/}"
err503File="${2##*/}"
shift
else
badParam dne "$@"
fi
else
badParam empty "$@"
fi
;;
-w|--webroot)
# path to webroot (copy 503)
if [ -n "$2" ]; then
if [ -d "$2" ]; then
webroot="${2%/}"
shift
else
badParam dne "$@"
fi
else
badParam empty "$@"
fi
;;
-a|--all)
# export ALL email stored in piler
exportAll=1
;;
--start)
# export starting at this date
if [ -n "$2" ]; then
exportStart="$2"
shift
else
badParam empty "$@"
fi
;;
--end)
# export ending at this date
if [ -n "$2" ]; then
exportEnd="$2"
shift
else
badParam empty "$@"
fi
;;
*)
printf "${err}\nUnknown option: %s\n" "$1"
printf "${cyan}Use '--help' for valid options.{$norm}\n\n"
exit 1
;;
esac
shift
done
### check pre-requisites and default values
# does the details file exist?
if [ ! -f "$configDetails" ]; then
badParam dne "(--details default)" "$configDetails"
fi
# is borg installed?
if ! command -v borg > /dev/null; then
printf "\n${err}ERROR: BORG is not installed on this system!${norm}\n\n"
exit 3
fi
# if 503 functionality is enabled, do 503 related files exist?
if [ "$use503" -eq 1 ]; then
if [ ! -f "$err503Path" ]; then
badParam dne "(--503-path default)" "$err503Path"
elif [ ! -d "$webroot" ]; then
badParam dne "(--webroot default)" "$webroot"
fi
fi
### start logging
printf "${magenta}[%s] --- Start %s execution ---${norm}\n" \
"$(stamp)" "$scriptName" >> "$logFile"
printf "${cyan}[%s] -- [INFO] Log located at ${yellow}%s${cyan} --${norm}\n" \
"$(stamp)" "$logFile" >> "$logFile"
### 503 functionality
if [ "$use503" -eq 1 ]; then
printf "${cyan}[%s] -- [INFO] Copying 503 error page to " \
"$(stamp)" >> "$logFile"
printf "webroot -- ${norm}\n" >> "$logFile"
if ! \cp --force "${err503Path}" "${webroot}/${err503File}" 2>> "$logFile"
then
printf "${warn}[%s] -- [WARNING] Failed to copy 503 error page. " \
"$(stamp)" >> "$logFile"
printf "Web users will NOT be notified --${norm}\n" >> "$logFile"
warnCount=$((warnCount+1))
else
printf "${ok}[%s] -- [SUCCESS] 503 error page copied --${norm}\n" \
"$(stamp)" >> "$logFile"
# set cleanup flag
err503Copied=1
fi
fi
### read details file to get variables needed to dump sql and run borg
# check if config details file was provided as a relative or absolute path
case "${configDetails}" in
/*)
# absolute path, no need to rewrite variable
. "${configDetails}"
;;
*)
# relative path, prepend './' to create absolute path
. "./${configDetails}"
;;
esac
printf "${cyan}[%s] -- [INFO] ${yellow}%s${cyan} imported --${norm}\n" \
"$(stamp)" "$configDetails" >> "$logFile"
## create tmp directory and change to it for export operations
printf "${cyan}[%s] -- [INFO] Exporting EML files from piler --${norm}\n" \
"$(stamp)" >> "$logFile"
# create temporary directory to dump files before borg backup
if ! exportDir=$( mktemp -d 2>>"$logFile" ); then
exitError 111 "Could not create temporary directory for exported EML files"
fi
if ! cd "$exportDir"; then
exitError 112 "Unable to change to temporary export directory"
fi
printf "${cyan}[%s] -- [INFO] EML files will be temporarily stored in:" \
"$(stamp)" >> "$logFile"
printf "\n${yellow}%s/${cyan} --${norm}\n" "$exportDir" >> "$logFile"
## export EML from piler
if [ "$exportAll" -eq 1 ]; then
if ! pilerexport -A 2>>"$logFile"; then
exitError 115 "There was a problem while exporting EML from piler"
fi
printf "${ok}[%s] -- [SUCCESS] Exported EML files from piler --${norm}\n" \
"$(stamp)" >> "$logFile"
else
if ! pilerexport -a "$exportStart" -b "$exportEnd" 2>>"$logFile"; then
exitError 115 "There was a problem while exporting EML from piler"
fi
printf "${ok}[%s] -- [SUCCESS] Exported EML files from piler --${norm}\n" \
"$(stamp)" >> "$logFile"
fi
### pre-backup tasks completed -- move to borg tasks
printf "${ok}[%s] -- [SUCCESS] Pre-backup tasks completed --${norm}\n" \
"$(stamp)" >> "$logFile"
### Run borg variable checks
printf "${cyan}[%s] -- [INFO] Verifying supplied borg details --${norm}\n" \
"$(stamp)" >> "$logFile"
## verify borg base directory
if [ -z "${borgBaseDir}" ]; then
badDetails empty 'borgBaseDir'
elif [ ! -d "${borgBaseDir}" ]; then
badDetails dne 'borgBaseDir'
fi
printf "${magenta}details:borgBaseDir ${norm}-- ${ok}[OK]${norm}\n" \
>> "$logFile"
export BORG_BASE_DIR="${borgBaseDir%/}"
## check path to SSH keyfile
if [ -z "${borgSSHKey}" ]; then
badDetails empty 'borgSSHKey'
elif [ ! -f "${borgSSHKey}" ]; then
badDetails dne 'borgSSHKey'
fi
printf "${magenta}details:borgSSHKey ${norm}-- ${ok}[OK]${norm}\n" \
>> "$logFile"
export BORG_RSH="ssh -i ${borgSSHKey}"
## check borg repo connect string
if [ -z "${borgConnectRepo}" ]; then
badDetails empty 'borgConnectRepo'
fi
printf "${magenta}details:borgConnectRepo ${norm}-- ${ok}[OK]${norm}\n" \
>> "$logFile"
export BORG_REPO="${borgConnectRepo}"
## check borg repo password
if [ -n "${borgRepoPassphrase}" ]; then
printf "${magenta}details:borgRepoPassphrase ${norm}-- ${ok}[OK]${norm}\n" \
>> "$logFile"
export BORG_PASSPHRASE="${borgRepoPassphrase}"
else
# if passwd is blank intentionally, this is insecure
printf "${warn} -- [WARNING] Using a borg repo without a password is an " \
>> "$logFile"
printf "insecure configuration --${norm}\n" >> "$logFile"
warnCount=$((warnCount+1))
# if this was an accident, we need to provide a bogus passwd so borg fails
# otherwise it will sit forever just waiting for input
export BORG_PASSPHRASE="DummyPasswordSoBorgFails"
fi
## export borg remote path, if specified
if [ -n "${borgRemote}" ]; then export BORG_REMOTE_PATH="${borgRemote}"; fi
## read additional files
if [ -n "${borgXtraListPath}" ]; then
# check if file actually exists
if [ ! -f "${borgXtraListPath}" ]; then
badDetails dne 'borgXtraListPath'
fi
# read file contents into concatenated list for echo to cmdline
while read -r xtraItem; do
if [ -z "${xtraList}" ]; then
xtraList="${xtraItem}"
else
xtraList="${xtraList} ${xtraItem}"
fi
done <<EOF
$( sed -e '/^\s*#.*$/d' -e '/^\s*$/d' "${borgXtraListPath}" )
EOF
printf "${magenta}details:borgXtraListPath ${norm}-- ${ok}[OK]${norm}\n" \
>> "$logFile"
includeXtra=1
fi
## check if exlusion list file is specified
if [ -n "${borgExcludeListPath}" ]; then
# check if the file actually exists
if [ ! -f "${borgExcludeListPath}" ]; then
badDetails dne 'borgExcludeListPath'
fi
exclusions=1
fi
### create borg temp dir:
## python requires a writable temporary directory when unpacking borg and
## executing commands. This defaults to /tmp but many systems mount /tmp with
## the 'noexec' option for security. Thus, we will use/create a 'tmp' folder
## within the BORG_BASE_DIR and instruct python to use that instead of /tmp
# check if BORG_BASE_DIR/tmp exists, if not, create it
if [ ! -d "${borgBaseDir}/tmp" ]; then
if ! mkdir "${borgBaseDir}/tmp"; then
exitError 132 "Unable to create borg ${borgBaseDir}/tmp directory"
else
printf "${cyan}[%s] -- [INFO] Created ${yellow}%s/tmp " \
"$(stamp)" "${borgBaseDir}" >> "$logFile"
printf "${cyan}--${norm}\n" >> "$logFile"
fi
fi
export TMPDIR="${borgBaseDir}/tmp"
### execute borg depending on whether extra files and/or exclusions are defined
## construct the proper borg commandline
# base command
if [ "$exclusions" -eq 0 ]; then
borgCMD="borg --show-rc create ${borgCreateParams} \
::$(date +%Y-%m-%d_%H%M%S) \
${exportDir}"
elif [ "$exclusions" -eq 1 ]; then
borgCMD="borg --show-rc create ${borgCreateParams} \
--exclude-from ${borgExcludeListPath} \
::$(date +%Y-%m-%d_%H%M%S) \
${exportDir}"
fi
# add extra locations if defined
if [ "$includeXtra" -eq 1 ]; then
borgCMD="${borgCMD} ${xtraList}"
fi
# execute borg
printf "${cyan}[%s] -- [INFO] Executing borg backup operation --${norm}\n" \
"$(stamp)" >> "$logFile"
${borgCMD} 2>> "$logFile"
borgResult="$?"
## check borg exit status
if [ "$borgResult" -eq 0 ]; then
printf "${ok}[%s] -- [SUCCESS] Borg backup completed " \
"$(stamp)" >> "$logFile"
printf "successfully --${norm}\n" >> "$logFile"
elif [ "$borgResult" -eq 1 ]; then
printf "${warn}[%s] -- [WARNING] Borg completed with warnings. " \
"$(stamp)" >> "$logFile"
printf "Review this logfile for details --${norm}\n" >> "$logFile"
warnCount=$((warnCount+1))
elif [ "$borgResult" -ge 2 ]; then
err_1="Borg exited with a critical error. Please review this log file"
err_2="for details."
exitError 138 "$err_1 $err_2"
else
printf "${warn}[%s] -- [WARNING] Borg exited with unknown return code. " \
"$(stamp)" >> "$logFile"
printf "Review this logfile for details --${norm}\n" >> "$logFile"
warnCount=$((warnCount+1))
fi
### execute borg prune if paramters are provided, otherwise skip with a warning
if [ -n "${borgPruneSettings}" ]; then
printf "${cyan}[%s] -- [INFO] Executing borg prune operation --${norm}\n" \
"$(stamp)" >> "$logFile"
borg prune --show-rc -v ${borgPruneParams} ${borgPruneSettings} \
2>> "$logFile"
borgPruneResult="$?"
else
printf "${warn}[%s] -- [WARNING] No prune parameters provided. " \
"$(stamp)" >> "$logFile"
printf "Your archive will continue growing with each backup --${norm}\n" \
>> "$logFile"
warnCount=$((warnCount+1))
fi
## report on prune operation if executed
if [ -n "${borgPruneResult}" ]; then
if [ "${borgPruneResult}" -eq 0 ]; then
printf "${ok}[%s] -- [SUCCESS] Borg prune completed --${norm}\n" \
"$(stamp)" >> "$logFile"
elif [ "$borgPruneResult" -eq 1 ]; then
printf "${warn}[%s] -- [WARNING] Borg prune completed with warnings. " \
"$(stamp)" >> "$logFile"
printf "Review this logfile for details --${norm}\n" >> "$logFile"
warnCount=$((warnCount+1))
elif [ "$borgPruneResult" -ge 2 ]; then
err_1="Borg prune exited with a critical error. Please review this"
err_2="log file for details."
exitError 139 "$err_1 $err_2"
else
printf "${warn}[%s] -- [WARNING] Borg prune exited with an unknown " \
"$(stamp)" >> "$logFile"
printf "return code. Review this logfile for details --${norm}\n" \
>> "$logFile"
warnCount=$((warnCount+1))
fi
fi
### all processes successfully completed, cleanup and exit gracefully
# note successful completion of borg commands
printf "${ok}[%s] -- [SUCCESS] Backup operations completed --${norm}\n" \
"$(stamp)" >> "$logFile"
# cleanup
cleanup
# note complete success, tally warnings and exit
printf "${ok}[%s] -- [SUCCESS] All processes completed successfully --" \
"${norm}\n" \
"$(stamp)" >> "$logFile"
printf "${magenta}[%s] --- %s execution completed ---${norm}\n" \
"$(stamp)" "$scriptName" >> "$logFile"
if [ "$warnCount" -gt 0 ]; then
printf "${warn}%s warnings issued!${norm}\n" "${warnCount}" >> "$logFile"
else
printf "${ok}0 warnings issued.${norm}\n" >> "$logFile"
fi
exit 0
### error codes
# 1: parameter error
# 2: not run as root
# 3: borg not installed
# 99: TERM signal trapped
# 111: could not create tmp dir for EML dump from piler-export
# 112: could not change to tmp dir to start export operation
# 115: problem during piler-export operation
# 130: null configuration variable in details file
# 131: invalid configuration variable in details file
# 138: borg exited with a critical error
# 139: borg prune exited with a critical error