From a5bc4549e9828958f8f33e55c1d291b3a6f5e526 Mon Sep 17 00:00:00 2001 From: Asif Bacchus Date: Wed, 6 May 2020 21:29:48 -0600 Subject: [PATCH] basic import from generic backup script --- backup/503_backup.html | 26 ++ backup/backup.details | 61 ++++ backup/backup.sh | 572 +++++++++++++++++++++++++++++++++++ backup/excludeLocations.borg | 2 + backup/xtraLocations.borg | 57 ++++ 5 files changed, 718 insertions(+) create mode 100644 backup/503_backup.html create mode 100644 backup/backup.details create mode 100644 backup/backup.sh create mode 100644 backup/excludeLocations.borg create mode 100644 backup/xtraLocations.borg diff --git a/backup/503_backup.html b/backup/503_backup.html new file mode 100644 index 0000000..ce13b9a --- /dev/null +++ b/backup/503_backup.html @@ -0,0 +1,26 @@ + + + + + + 503 - Unavailable: Backup in progress + + + + + + +

Bad timing!

+

Seems you're trying to access me during my daily backup window. Don't worry though, I should be up and running again very soon.

+

My average backup window duration is pretty short and I'm quite busy during that time copying your super-important stuff to my secure hiding place so they stay safe in case anything ever happens to me!

+

I'm really sorry for the delay. Please try me again soon!

+ + + \ No newline at end of file diff --git a/backup/backup.details b/backup/backup.details new file mode 100644 index 0000000..25970fc --- /dev/null +++ b/backup/backup.details @@ -0,0 +1,61 @@ +####### +### backup script configuration details +### +### This file contains sensitive information, make sure you have protected +### it by restricting permissions! +### Run the following in the directory where this file is located: +### chown root:root ./backup.details +### chmod 600 ./backup.details +### +### Do NOT include any commands in this file as they WILL be executed!!! +####### + + +### borg details +# if you're unsure what to enter here, please consult the repo wiki and/or +# the borg documentation + +# base configuration directory for borg, all borg parameters use this directory +# as their 'root'. I recommend setups with this being "/var/borgbackup", the +# default is "$HOME" or "~$USER" in that order. If you're unsure, try "$HOME" +borgBaseDir="/var/borgbackup" + +# full path to the SSH key used to connect to your remote backup server +borgSSHKey="/var/borgbackup/private.key" + +# connection string to access the borg repo on your remote backup server +# this is usually in the form user@servername.tld:repoName/ +borgConnectRepo="jdoe123@borg.server.net:mailcow/" + +# password to access repo +# this was set when the repo was initialized and, while optional, is HIGHLY +# recommended for security +borgRepoPassphrase="p@ssW0rd" + +# keyfile to access repo +# FULL PATH where the associated keyfile for your repo is located -- relevant +# only if your repo requires a keyfile (i.e. 'keyfile' vs 'repokey') and if you +# are not using the default keyfile location +borgKeyfileLocation="/var/borgbackup/.config/borg/keys/server_address__repo_name" + +# REQUIRED: path to text file containing a list (one per line) of files/ +# directories to include in your backup. Since this is a generic backup script, +# nothing is defined by default. Therefore, ONLY files specified in this file +# will be backed up! +# see repo wiki for more details +borgXtraListPath="/root/scripts/backup/xtraLocations.borg" + +# OPTIONAL: path to file containing files/directories or 'patterns' to be +# excluded in a BORG RECOGNIZED format +# see repo wiki for more details or consult borg documentation +# leave blank for no exclusions. +borgExcludeListPath="/root/scripts/backup/excludeLocations.borg" + +# parameters to determine how borg deletes aged backups +# more details in the repo wiki and/or borg documentation +# leave blank to skip pruning altogether -- NOT recommended! +borgPruneSettings="--keep-within=14d --keep-daily=30 --keep-weekly=12 --keep-monthly=12" + +# location of borg instance on your remote backup server +# this is very often just "borg1" +borgRemote="borg1" \ No newline at end of file diff --git a/backup/backup.sh b/backup/backup.sh new file mode 100644 index 0000000..b044c29 --- /dev/null +++ b/backup/backup.sh @@ -0,0 +1,572 @@ +#!/bin/sh + +####### +### mailcow backup using borgbackup +### this assumes three things: +### 1. standard mailcow-dockerized setup as per the docs +### 2. using borg to perform backups to ssh-capable remote server +### 3. remote repo already set-up and configured +####### + + +### text formatting presents +if command -v tput > /dev/null; then + bold=$(tput bold) + cyan=$(tput setaf 6) + err=$(tput bold)$(tput setaf 1) + magenta=$(tput setaf 5) + norm=$(tput sgr0) + ok=$(tput setaf 2) + warn=$(tput bold)$(tput setaf 3) + width=$(tput cols) + yellow=$(tput setaf 3) +else + bold="" + cyan="" + err="" + magenta="" + norm="" + ok="" + warn="" + width=80 + yellow="" +fi + + +### trap +trap trapExit 1 2 3 6 + + +### functions + +# bad configuration value passed in details file +badDetails () { + if [ "$1" = "empty" ]; then + exitError 130 "details:${2} cannot be NULL (undefined)" + elif [ "$1" = "dne" ]; then + exitError 131 "details:${2} file or directory does not exist." + fi +} + +# bad parameter passed to script +badParam () { + if [ "$1" = "dne" ]; then + printf "\n%sError: '%s %s'\n" "$err" "$2" "$3" + printf "file or directory does not exist.%s\n\n" "$norm" + exit 1 + elif [ "$1" = "empty" ]; then + printf "\n%sError: '%s' cannot have a NULL (empty) value.\n" "$err" "$2" + printf "%sPlease use '--help' for assistance%s\n\n" "$cyan" "$norm" + exit 1 + elif [ "$1" = "svc" ]; then + printf "\n%sError: '%s %s': Service does not exist!%s\n\n" \ + "$err" "$2" "$3" "$norm" + exit 1 + elif [ "$1" = "user" ]; then + printf "\n%sError: '%s %s': User does not exist!%s\n\n" \ + "$err" "$2" "$3" "$norm" + exit 1 + fi +} + +# cleanup +cleanup () { + # cleanup 503 if copied + if [ "$err503Copied" -eq 1 ]; then + if ! rm -f "$webroot/$err503File" 2>>"$logFile"; then + printf "%s[%s] -- [WARNING] Could not remove 503 error page." \ + "$warn" "$(stamp)" >> "$logFile" + printf " Web interface will not function until this file is " \ + >> "$logFile" + printf "removed --%s\n" "$norm" >> "$logFile" + warnCount=$((warnCount+1)) + else + printf "%s[%s] -- [INFO] 503 error page removed --%s\n" \ + "$cyan" "$(stamp)" "$norm" >> "$logFile" + fi + fi +} + +# call cleanup and then exit with error report +exitError () { + printf "%s[%s] -- [ERROR] %s: %s --%s\n" \ + "$err" "$(stamp)" "$1" "$2" "$norm" >> "$logFile" + cleanup + # note script completion with error + printf "%s[%s] --- %s execution completed with error ---%s\n" \ + "$err" "$(stamp)" "$scriptName" "$norm" >> "$logFile" + exit "$1" +} + +# display script help information +scriptHelp () { + newline + printf "%sUsage: %s [parameters]%s\n\n" "$bold" "$scriptName" "$norm" + textblock "There are NO mandatory parameters. If a parameter is not supplied, its default value will be used. In the case of a switch parameter, it will remain DEactivated if NOT specified." + newline + textblock "Switches are listed then followed by a description of their effect on the following line. Finally, if a default value exists, it will be listed on the next line in (parentheses)." + newline + textblock "${magenta}--- script related parameters ---${norm}" + newline + switchTextblock "-c | --config | --details" + textblock "Path to the configuration key/value-pair file for this script." + defaultsTextblock "(scriptPath/scriptName.details)" + newline + switchTextblock "-h | -? | --help" + textblock "This help screen" + newline + switchTextblock "-l | --log" + textblock "Path to write log file" + defaultsTextblock "(scriptPath/scriptName.log)" + newline + switchTextblock "[SWITCH] -v | --verbose" + textblock "Log borg output with increased verbosity (list all files). Careful! Your log file can get very large very quickly!" + defaultsTextblock "(normal output, option is OFF)" + newline + textblock "${magenta}--- 503 functionality ---${norm}" + newline + switchTextblock "[SWITCH] -5 | --use-503" + textblock "Copy an 'error 503' page/indicator file to your webroot for your webserver to find. Specifying this option will enable other 503 options." + defaultsTextblock "(do NOT copy, option is OFF)" + newline + switchTextblock "--503-path" + textblock "Path to the file you want copied to your webroot as the 'error 503' page." + defaultsTextblock "(scriptPath/503_backup.html)" + newline + switchTextblock "-w | --webroot" + textblock "Path to where the 'error 503' file should be copied." + defaultsTextblock "(/usr/share/nginx/html/)" + newline + textblock "More details and examples of script usage can be found in the repo wiki at ${yellow}https://git.asifbacchus.app/asif/myGitea/wiki${norm}" + newline +} + +# generate dynamic timestamps +stamp () { + (date +%F" "%T) +} + +textblock() { + printf "%s\n" "$1" | fold -w "$width" -s +} + +defaultsTextblock() { + printf "%s%s%s\n" "$yellow" "$1" "$norm" +} + +switchTextblock() { + printf "%s%s%s\n" "$cyan" "$1" "$norm" +} + +# print a blank line +newline () { + printf "\n" +} + +# same as exitError but for signal captures +trapExit () { + printf "%s[%s] -- [ERROR] 99: Caught signal --%s\n" \ + "$err" "$(stamp)" "$norm" >> "$logFile" + cleanup + # note script completion with error + printf "%s[%s] --- %s execution was terminated via signal ---%s\n" \ + "$err" "$(stamp)" "$scriptName" "$norm" >> "$logFile" + exit 99 +} + +### end of functions + + +### default variable values + +## script related +# store logfile in the same directory as this script file using the same file +# name as the script but with the extension '.log' +scriptPath="$( CDPATH='' cd -- "$( dirname -- "$0" )" && pwd -P )" +scriptName="$( basename "$0" )" +logFile="$scriptPath/${scriptName%.*}.log" +warnCount=0 +configDetails="$scriptPath/${scriptName%.*}.details" +err503Copied=0 +exclusions=0 +# borg output verbosity -- normal +borgCreateParams='--stats' +borgPruneParams='--list' + +# 503 related +use503=0 +err503Path="$scriptPath/503_backup.html" +err503File="${err503Path##*/}" +webroot="/usr/share/nginx/html" + + +### process startup parameters +while [ $# -gt 0 ]; do + case "$1" in + -h|-\?|--help) + # display help + scriptHelp + exit 0 + ;; + -l|--log) + # set log file location + if [ -n "$2" ]; then + logFile="${2%/}" + shift + else + badParam empty "$@" + fi + ;; + -v|--verbose) + # set verbose logging from borg + borgCreateParams='--list --stats' + borgPruneParams='--list' + ;; + -c|--config|--details) + # location of config details file + if [ -n "$2" ]; then + if [ -f "$2" ]; then + configDetails="${2%/}" + shift + else + badParam dne "$@" + fi + else + badParam empty "$@" + fi + ;; + -5|--use-503) + # enable copying 503 error page to webroot + use503=1 + ;; + --503-path) + # FULL path to 503 file + if [ -n "$2" ]; then + if [ -f "$2" ]; then + err503Path="${2%/}" + err503File="${2##*/}" + shift + else + badParam dne "$@" + fi + else + badParam empty "$@" + fi + ;; + -w|--webroot) + # path to webroot (copy 503) + if [ -n "$2" ]; then + if [ -d "$2" ]; then + webroot="${2%/}" + shift + else + badParam dne "$@" + fi + else + badParam empty "$@" + fi + ;; + *) + printf "\n%sUnknown option: %s\n" "$err" "$1" + printf "%sUse '--help' for valid options.%s\n\n" "$cyan" "$norm" + exit 1 + ;; + esac + shift +done + + +### check pre-requisites and default values +# check if running as root, otherwise exit +if [ "$( id -u )" -ne 0 ]; then + printf "\n%sERROR: script MUST be run as ROOT%s\n\n" "$err" "$norm" + exit 2 +fi +# does the details file exist? +if [ ! -f "$configDetails" ]; then + badParam dne "(--details default)" "$configDetails" +fi +# is borg installed? +if ! command -v borg > /dev/null; then + printf "\n%sERROR: BORG is not installed on this system!%s\n\n" "$err" "$norm" + exit 3 +fi +# if 503 functionality is enabled, do 503 related files exist? +if [ "$use503" -eq 1 ]; then + if [ ! -f "$err503Path" ]; then + badParam dne "(--503-path default)" "$err503Path" + elif [ ! -d "$webroot" ]; then + badParam dne "(--webroot default)" "$webroot" + fi +fi + + +### start logging +printf "%s[%s] --- Start %s execution ---%s\n" \ + "$magenta" "$(stamp)" "$scriptName" "$norm" >> "$logFile" +printf "%s[%s] -- [INFO] Log located at %s%s%s --%s\n" \ + "$cyan" "$(stamp)" "$yellow" "$logFile" "$cyan" "$norm" >> "$logFile" + + +### 503 functionality +if [ "$use503" -eq 1 ]; then + printf "%s[%s] -- [INFO] Copying 503 error page to " \ + "$cyan" "$(stamp)" >> "$logFile" + printf "webroot -- %s\n" "$norm">> "$logFile" + if ! cp --force "${err503Path}" "${webroot}/${err503File}" 2>> "$logFile" + then + printf "%s[%s] -- [WARNING] Failed to copy 503 error page. " \ + "$warn" "$(stamp)" >> "$logFile" + printf "Web users will NOT be notified --%s\n" "$norm" >> "$logFile" + warnCount=$((warnCount+1)) + else + printf "%s[%s] -- [SUCCESS] 503 error page copied --%s\n" \ + "$ok" "$(stamp)" "$norm" >> "$logFile" + # set cleanup flag + err503Copied=1 + fi +fi + + +### read details file to get variables needed run borg +# check if config details file was provided as a relative or absolute path +case "${configDetails}" in + /*) + # absolute path, no need to rewrite variable + . "${configDetails}" + ;; + *) + # relative path, prepend './' to create absolute path + . "./${configDetails}" + ;; +esac +printf "%s[%s] -- [INFO] %s%s%s imported --%s\n" \ + "$cyan" "$(stamp)" "$yellow" "$configDetails" "$cyan" "$norm" >> "$logFile" + + +### Run borg variable checks +printf "%s[%s] -- [INFO] Verifying supplied borg details --%s\n" \ + "$cyan" "$(stamp)" "$norm" >> "$logFile" + +## read additional files -- this is required otherwise nothing to backup! +if [ -z "${borgXtraListPath}" ]; then + badDetails empty 'xtraLocations' +else + # check if file actually exists + if [ ! -f "${borgXtraListPath}" ]; then + badDetails dne 'borgXtraListPath' + fi + # read file contents into concatenated list for echo to cmdline + while read -r xtraItem; do + if [ -z "${xtraList}" ]; then + xtraList="${xtraItem}" + else + xtraList="${xtraList} ${xtraItem}" + fi + done <> "$logFile" +fi + +## verify borg base directory +if [ -z "${borgBaseDir}" ]; then + badDetails empty 'borgBaseDir' +elif [ ! -d "${borgBaseDir}" ]; then + badDetails dne 'borgBaseDir' +fi +printf "%sdetails:borgBaseDir %s-- %s[OK]%s\n" \ + "$magenta" "$norm" "$ok" "$norm" >> "$logFile" +export BORG_BASE_DIR="${borgBaseDir%/}" + +## check path to SSH keyfile +if [ -z "${borgSSHKey}" ]; then + badDetails empty 'borgSSHKey' +elif [ ! -f "${borgSSHKey}" ]; then + badDetails dne 'borgSSHKey' +fi +printf "%sdetails:borgSSHKey %s-- %s[OK]%s\n" \ + "$magenta" "$norm" "$ok" "$norm" >> "$logFile" +export BORG_RSH="ssh -i ${borgSSHKey}" + +## check borg repo connect string +if [ -z "${borgConnectRepo}" ]; then + badDetails empty 'borgConnectRepo' +fi +printf "%sdetails:borgConnectRepo %s-- %s[OK]%s\n" \ + "$magenta" "$norm" "$ok" "$norm" >> "$logFile" +export BORG_REPO="${borgConnectRepo}" + +## check borg repo password +if [ -n "${borgRepoPassphrase}" ]; then + printf "%sdetails:borgRepoPassphrase %s-- %s[OK]%s\n" \ + "$magenta" "$norm" "$ok" "$norm" >> "$logFile" + export BORG_PASSPHRASE="${borgRepoPassphrase}" +else + # if passwd is blank intentionally, this is insecure + printf "%s-- [WARNING] Using a borg repo without a password is an " \ + "$warn" >> "$logFile" + printf "insecure configuration --%s\n" "$norm">> "$logFile" + warnCount=$((warnCount+1)) + # if this was an accident, we need to provide a bogus passwd so borg fails + # otherwise it will sit forever just waiting for input + export BORG_PASSPHRASE="DummyPasswordSoBorgFails" +fi + +## check borg repository keyfile location +if [ -z "${borgKeyfileLocation}" ]; then + printf "%sdetails:borgKeyfileLocation %s-- %s[DEFAULT]%s\n" "$magenta" "$norm" "$ok" "$norm" >> "$logFile" +else + # check if keyfile location exists + if [ ! -f "${borgKeyfileLocation}" ]; then + badDetails dne 'borgKeyfileLocation' + fi + printf "%sdetails:borgKeyfileLocation %s-- %s[OK]%s\n" "$magenta" "$norm" "$ok" "$norm" >> "$logFile" + export BORG_KEY_FILE="${borgKeyfileLocation}" +fi + +## export borg remote path, if specified +if [ -n "${borgRemote}" ]; then export BORG_REMOTE_PATH="${borgRemote}"; fi + +## check if exlusion list file is specified +if [ -n "${borgExcludeListPath}" ]; then + # check if the file actually exists + if [ ! -f "${borgExcludeListPath}" ]; then + badDetails dne 'borgExcludeListPath' + fi +exclusions=1 +fi + + +### create borg temp dir: +## python requires a writable temporary directory when unpacking borg and +## executing commands. This defaults to /tmp but many systems mount /tmp with +## the 'noexec' option for security. Thus, we will use/create a 'tmp' folder +## within the BORG_BASE_DIR and instruct python to use that instead of /tmp + +# check if BORG_BASE_DIR/tmp exists, if not, create it +if [ ! -d "${borgBaseDir}/tmp" ]; then + if ! mkdir "${borgBaseDir}/tmp"; then + exitError 132 "Unable to create borg ${borgBaseDir}/tmp directory" + else + printf "%s[%s] -- [INFO] Created %s%s/tmp " \ + "$cyan" "$(stamp)" "$yellow" "${borgBaseDir}" >> "$logFile" + printf "%s--%s\n" "$cyan" "$norm">> "$logFile" + fi +fi +export TMPDIR="${borgBaseDir}/tmp" + + +### execute borg depending on whether exclusions are defined + +## construct the proper borg commandline +# base command +if [ "$exclusions" -eq 0 ]; then + borgCMD="borg --show-rc create ${borgCreateParams} \ + ::$(date +%Y-%m-%d_%H%M%S) \ + ${xtraList}" +elif [ "$exclusions" -eq 1 ]; then + borgCMD="borg --show-rc create ${borgCreateParams} \ + --exclude-from ${borgExcludeListPath} \ + ::$(date +%Y-%m-%d_%H%M%S) \ + ${xtraList}" +fi + +# execute borg +printf "%s[%s] -- [INFO] Executing borg backup operation --%s\n" \ + "$cyan" "$(stamp)" "$norm" >> "$logFile" +${borgCMD} 2>> "$logFile" +borgResult="$?" + +## check borg exit status +if [ "$borgResult" -eq 0 ]; then + printf "%s[%s] -- [SUCCESS] Borg backup completed --%s\n" \ + "$ok" "$(stamp)" "$norm" >> "$logFile" +elif [ "$borgResult" -eq 1 ]; then + printf "%s[%s] -- [WARNING] Borg completed with warnings. " \ + "$warn" "$(stamp)" >> "$logFile" + printf "Review this logfile for details --%s\n" "$norm">> "$logFile" + warnCount=$((warnCount+1)) +elif [ "$borgResult" -ge 2 ]; then + err_1="Borg exited with a critical error. Please review this log file" + err_2="for details." + exitError 138 "$err_1 $err_2" +else + printf "%s[%s] -- [WARNING] Borg exited with unknown return code. " \ + "$warn" "$(stamp)" >> "$logFile" + printf "Review this logfile for details --%s\n" "$norm">> "$logFile" + warnCount=$((warnCount+1)) +fi + + +### execute borg prune if paramters are provided, otherwise skip with a warning +if [ -n "${borgPruneSettings}" ]; then + printf "%s[%s] -- [INFO] Executing borg prune operation --%s\n" \ + "$cyan" "$(stamp)" "$norm" >> "$logFile" + borg prune --show-rc -v ${borgPruneParams} ${borgPruneSettings} \ + 2>> "$logFile" + borgPruneResult="$?" +else + printf "%s[%s] -- [WARNING] No prune parameters provided. " \ + "$warn" "$(stamp)" >> "$logFile" + printf "Your archive will continue growing with each backup --%s\n" \ + "$norm" >> "$logFile" + warnCount=$((warnCount+1)) +fi + +## report on prune operation if executed +if [ -n "${borgPruneResult}" ]; then + if [ "${borgPruneResult}" -eq 0 ]; then + printf "%s[%s] -- [SUCCESS] Borg prune completed --%s\n" \ + "$ok" "$(stamp)" "$norm" >> "$logFile" + elif [ "$borgPruneResult" -eq 1 ]; then + printf "%s[%s] -- [WARNING] Borg prune completed with warnings. " \ + "$warn" "$(stamp)" >> "$logFile" + printf "Review this logfile for details --%s\n" "$norm" >> "$logFile" + warnCount=$((warnCount+1)) + elif [ "$borgPruneResult" -ge 2 ]; then + err_1="Borg prune exited with a critical error. Please review this" + err_2="log file for details." + exitError 139 "$err_1 $err_2" + else + printf "%s[%s] -- [WARNING] Borg prune exited with an unknown " \ + "$warn" "$(stamp)" >> "$logFile" + printf "return code. Review this logfile for details --%s\n" \ + "$norm" >> "$logFile" + warnCount=$((warnCount+1)) + fi +fi + + +### all processes successfully completed, cleanup and exit gracefully + +# note successful completion of borg commands +printf "%s[%s] -- [SUCCESS] Backup operations completed --%s\n" \ + "$ok" "$(stamp)" "$norm" >> "$logFile" + +# cleanup +cleanup + +# note complete success, tally warnings and exit +printf "%s[%s] -- [SUCCESS] All processes completed --%s\n" \ + "$ok" "$(stamp)" "$norm" >> "$logFile" +printf "%s[%s] --- %s execution completed ---%s\n" \ + "$magenta" "$(stamp)" "$scriptName" "$norm" >> "$logFile" +if [ "$warnCount" -gt 0 ]; then + printf "%s%s warnings issued!%s\n" "$warn" "${warnCount}" "$norm" >> "$logFile" +else + printf "%s0 warnings issued.%s\n" "$ok" "$norm" >> "$logFile" +fi +exit 0 + + +### error codes +# 1: parameter error +# 2: not run as root +# 3: borg not installed +# 99: TERM signal trapped +# 130: null configuration variable in details file +# 131: invalid configuration variable in details file +# 138: borg exited with a critical error +# 139: borg prune exited with a critical error \ No newline at end of file diff --git a/backup/excludeLocations.borg b/backup/excludeLocations.borg new file mode 100644 index 0000000..1a79084 --- /dev/null +++ b/backup/excludeLocations.borg @@ -0,0 +1,2 @@ + +EOF \ No newline at end of file diff --git a/backup/xtraLocations.borg b/backup/xtraLocations.borg new file mode 100644 index 0000000..0caaf1c --- /dev/null +++ b/backup/xtraLocations.borg @@ -0,0 +1,57 @@ +# Files and directories listed here will be included in your borg backup +# +# Good candidates for inclusion would be things like your mailcow configuration +# files, customized docker-compose overrides, your SSL certificates, etc. +# +# List the path to files/directories one per line. +# Any blank lines will be ignored. +# Any lines starting with '#' will be ignored as a comment. +# For consistency, you should include the trailing slash for directories. + +# these examples are for a very basic Debian machine hosting mailcow + + +### important system configuration files + +# basic configuration +/etc/fstab +/etc/network/interfaces +/etc/network/interfaces.d/ +/etc/systemd/timesyncd.conf + +# ssh configuration and host keys +/etc/ssh/ + +# apt configuration +/etc/apt/sources.list +/etc/apt/sources.list.d/ +/etc/apt/listchanges.conf +/etc/apt/apt.conf.d/50unattended-upgrades +/etc/apt/apt.conf.d/20auto-upgrades + +# user profile defaults and configurations +/etc/profile +/etc/bash.bashrc +/etc/skel/ +/etc/nanorc + +# selected root user files +/root/.bashrc +/root/.ssh/ + +# scripts +/scripts/ + + +### important programs and configurations + +# name of program for reference +# include the paths to important configuration files/directories and/or +# data directories + +# NGINX (example) +/etc/nginx/ +/usr/share/nginx/html/ + +# LetsEncrypt (example) +/etc/letsencrypt/