refactor(log): Use CLEF for JSON logs

Use Compact Log Event Format (CLEF) for JSON logs along with custom properties.

Folder.DotSettings

@@ -0,0 +1,4 @@
+<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=cfddns/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=mydomain/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=myserver/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

cfddns.sh

@@ -35,13 +35,19 @@ badParam() {
   if [ "$1" = "null" ]; then
     printf "\n%sERROR: '%s' cannot have a NULL (empty) value.\n" "$err" "$2"
     printf "%sPlease use '--help' for assistance.%s\n\n" "$cyan" "$norm"
+    ([ "$logToConsole" -eq 0 ]) && writeLog \
+      "${2} cannot have a NULL (empty) value" "critical" "fail" "preexec" 1
     exit 1
   elif [ "$1" = "dne" ]; then
     printf "\n%sERROR: '%s %s'\n" "$err" "$2" "$3"
     printf "file or directory does not exist or is empty.%s\n\n" "$norm"
+    ([ "$logToConsole" -eq 0 ]) && writeLog \
+      "${3}: file or directory does not exist or is empty" "critical" "fail" "preexec" 1
     exit 1
   elif [ "$1" = "errMsg" ]; then
     printf "\n%sERROR: %s%s\n\n" "$err" "$2" "$norm"
+    ([ "$logToConsole" -eq 0 ]) && writeLog \
+      "${2%%\Exit*}" "fatal" "fail" "preexec" 1
     exit 1
   fi
 }
@@ -79,7 +85,7 @@ listCFErrors() {
     messages="${messages#*_}"
 
     # output to log
-    writePlainTextLog "${message}" "err" "fail" "$2" "$code"
+    writeLog "${message}" "err" "fail" "$2:CF-ERR" "$code"
   done
 }
 
@@ -107,6 +113,31 @@ standardizeLogLevels() {
   esac
 }
 
+# $1: message, $2: level, $3: status, [$4: operation], [$5: code]
+writeJsonLog() {
+  # not enough information to generate a meaningful log message
+  if [ -z "$1" ] || [ $# -lt 3 ]; then
+    return
+  fi
+
+  JSON_PROPS="$(
+    jq --null-input -c \
+      --arg timestamp "$(getTimeStamp)" \
+      --arg hostname "$(getHostname)" \
+      --arg pName "$LOG_PROGRAM_NAME" \
+      --arg pid "$$" \
+      --arg level "$(standardizeLogLevels "$2")" \
+      --arg status "$(uppercase "$3")" \
+      --arg message "$1" \
+      --arg operation "${4:-UNSPECIFIED}" \
+      --arg code "${5:-0}" \
+      '{"@t":$timestamp, "@m":$message, "@l":$level, "hostname":$hostname, "programName":$pName, "pid":$pid, "status":$status, "operation":$operation, "code":$code}' \
+  )"
+
+  printf "%s\n" "$JSON_PROPS" >>"$logFile"
+  return
+}
+
 # modified syslog fmt: <ISO-8601 timestamp> <hostname> <tag>[pid]: [LEVEL:$2] [STATUS:$3] <MESSAGE:$1> (<OPERATION:$4>:<CODE:$5>)
 writePlainTextLog() {
   # not enough information to generate a meaningful log message
@@ -120,6 +151,17 @@ writePlainTextLog() {
     >>"$logFile"
 }
 
+writeLog() {
+  ([ "$logToNone" -eq 1 ]) && return
+
+  PT_LOG_LEVEL="$(standardizeLogLevels "$2")"
+  ([ "$logDebug" -eq 0 ] && [ "$PT_LOG_LEVEL" = "DEBUG" ]) && return
+
+  ([ "$logToJournal" -eq 1 ]) && return # TODO: replace this with function when implemented
+  ([ "$useJsonLogFmt" -eq 1 ]) && writeJsonLog "$@"
+  ([ "$useSyslogLogFmt" -eq 1 ]) && writePlainTextLog "$@"
+}
+
 scriptExamples() {
   newline
   printf "Update Cloudflare DNS host A/AAAA records with current IP address.\n"
@@ -153,34 +195,42 @@ scriptHelp() {
   textBlock "The only required parameter is '--records' which is a comma-delimited list of hostnames to update. However, there are several other options which may be useful to implement."
   textBlock "Parameters are listed below and followed by a description of their effect. If a default value exists, it will be listed on the following line in (parentheses)."
   newline
-  textBlock "${magenta}--- script related parameters ---${norm}"
+  textBlock "${magenta}--- script-related parameters ---${norm}"
   newline
   textBlockSwitches "-c | --cred | --creds | --credentials | -f (deprecated, backward-compatibility)"
-  textBlock "Path to file containing your Cloudflare *token* credentials. Please refer to the repo README for more information on format, etc."
+  textBlock "Path to the file containing your Cloudflare *token* credentials. Please refer to the repo README for more information on format, etc."
   textBlockDefaults "(${accountFile})"
   newline
-  textBlockSwitches "-l | --log"
+  textBlockSwitches "--log-file"
-  textBlock "Path where the log file should be written."
+  textBlock "Path where the log file should be written. Script will use file-based logging."
   textBlockDefaults "(${logFile})"
   newline
-  textBlockSwitches "--nc | --no-color | --no-colour"
-  textBlock "Switch value. Disables ANSI colours in the log. Useful if you review the logs using a reader that does not parse ANSI colour codes."
-  textBlockDefaults "(disabled: print logs in colour)"
-  newline
   textBlockSwitches "--log-console"
-  textBlock "Switch value. Output log to console (stdout) instead of a log file. Can be combined with --nc if desired."
+  textBlock "Switch value. Script will use console-based (stdout) logging."
-  textBlockDefaults "(disabled: output to log file)"
+  textBlockDefaults "(disabled: use file-based logging)"
   newline
   textBlockSwitches "--log-journal"
-  textBlock "Switch value. Output a structured log entry to your journaling system instead of a log file. Implies --nc."
+  textBlock "Switch value. Script will use journald-based logging."
-  textBlockDefaults "(disabled: output to log file)"
+  textBlockDefaults "(disabled: use file-based logging)"
   newline
-  textBlockSwitches "--no-log"
+  textBlockSwitches "--log-none"
-  textBlock "Switch value. Do not create a log (i.e. no console, no file). You will not have *any* output from the script if you choose this option, so you will not know if updates succeeded or failed."
+  textBlock "Switch value. Script will not log anything. You will not have *any* output from the script if you choose this option, so you will not know if updates succeed or fail."
-  textBlockDefaults "(disabled: output to log file)"
+  textBlockDefaults "(disabled: use file-based logging)"
+  newline
+  textBlockSwitches "--log-debug"
+  textBlock "Switch value. Log 'debug' level messages. This should normally remain disabled."
+  textBlockDefaults "(disabled: do not log 'debug' messages)"
+  newline
+  textBlockSwitches "--fmt-json"
+  textBlock "Switch value. Use JSON formatted key-value pair structured logging format. Best when using a log parsing tool or log management system to read your log files. Does not apply when using '--log-journal'."
+  textBlockDefaults "(enabled)"
+  newline
+  textBlockSwitches "--fmt-syslog"
+  textBlock "Switch value. Use a modified syslog (plain-text) format for log messages. Ideal when logging to the console. Does not apply when using '--log-journal'."
+  textBlockDefaults "(disabled: use structured JSON logging)"
   newline
   textBlockSwitches "-h | --help | -?"
-  textBlock "Display this help screen."
+  textBlock "Display this information screen."
   newline
   textBlockSwitches "--examples"
   textBlock "Show some usage examples."
@@ -191,7 +241,7 @@ scriptHelp() {
   textBlock "Comma-delimited list of hostnames for which IP addresses should be updated in Cloudflare DNS. This parameter is REQUIRED. Note that this script will only *update* records, it will not create new ones. If you supply hostnames that are not already defined in DNS, the script will log a warning and will skip those hostnames."
   newline
   textBlockSwitches "-i | --ip | --ip-address | -a | --address"
-  textBlock "New IP address for DNS host records. If you omit this, the script will attempt to auto-detect your public IP address and use that."
+  textBlock "New IP address for DNS host records. If you omit this, the script will attempt to auto-detect your public IP address and use that. N.B. IP addresses are *not* parsed for correctness."
   newline
   textBlockSwitches "-4 | --ip4 | --ipv4"
   textBlock "Switch value. Update Host 'A' records (IP4) only. Note that this script can only update either A *or* AAAA records. If you need to update both, you'll have to run the script once in IP4 mode and again in IP6 mode. If you specify both this switch and the IP6 switch, the last one specified will take effect."
@@ -223,19 +273,24 @@ textBlockSwitches() {
 }
 
 ### default variable values
+LOG_PROGRAM_NAME="CFDDNS"
 scriptPath="$(CDPATH='' \cd -- "$(dirname -- "$0")" && pwd -P)"
 scriptName="$(basename "$0")"
 logFile="$scriptPath/${scriptName%.*}.log"
 logToJournal=0
+logToConsole=0
+logToNone=0
+logDebug=0
+useJsonLogFmt=1
+useSyslogLogFmt=0
 accountFile="$scriptPath/cloudflare.credentials"
-colourizeLogFile=1
 dnsRecords=""
 dnsSeparator=","
 ipAddress=""
 ip4=1
 ip6=0
-ip4DetectionSvc="http://ipv4.icanhazip.com"
+ip4DetectionSvc="https://ipv4.icanhazip.com"
-ip6DetectionSvc="http://ipv6.icanhazip.com"
+ip6DetectionSvc="https://ipv6.icanhazip.com"
 invalidDomainCount=0
 failedHostCount=0
 
@@ -253,8 +308,8 @@ while [ $# -gt 0 ]; do
   # display sample commands
     scriptExamples
     ;;
-  -l | --log)
+  --log-file)
-  # set log file location
+  # use file-based logging at the specified location
     if [ -n "$2" ]; then
       logFile="${2%/}"
       shift
@@ -263,23 +318,35 @@ while [ $# -gt 0 ]; do
     fi
     ;;
   --log-console)
-  # log to the console instead of a file
+  # use console-based logging
     logFile="/dev/stdout"
+    logToConsole=1
     ;;
   --log-journal)
-  # log to the journal system instead of a file
+  # use journald-based logging
     logToJournal=1
     ;;
-  --no-log)
+  --log-none)
   # do not log anything
     logFile="/dev/null"
+    logToNone=1
     ;;
-  --nc | --no-color | --no-colour)
+  --log-debug)
-  # do not colourize log file
+  # log debugging messages
-    colourizeLogFile=0
+    logDebug=1
+    ;;
+  --fmt-json)
+  # use JSON log formatting
+    useJsonLogFmt=1
+    useSyslogLogFmt=0
+    ;;
+  --fmt-syslog)
+  # use modified syslog plain-text log formatting
+    useSyslogLogFmt=1
+    useJsonLogFmt=0
     ;;
   -c | --cred* | -f)
-  # path to Cloudflare credentials file
+  # path to the Cloudflare credentials file
     if [ -n "$2" ]; then
       if [ -f "$2" ] && [ -s "$2" ]; then
         accountFile="${2%/}"
@@ -322,6 +389,8 @@ while [ $# -gt 0 ]; do
   *)
     printf "\n%sUnknown option: %s\n" "$err" "$1"
     printf "%sUse '--help' for valid options.%s\n\n" "$cyan" "$norm"
+    ([ "$logToConsole" -eq 0 ]) && writeLog \
+      "Unknown script parameter: '${1}'" "critical" "fail" "preexec" 1
     exit 1
     ;;
   esac
@@ -330,49 +399,36 @@ done
 
 ### pre-flight checks
 if ! command -v curl >/dev/null; then
-  printf "\n%sThis script requires 'curl' be installed and accessible. Exiting.%s\n\n" "$err" "$norm"
+  printf "\n%sThis script requires that 'curl' is installed and accessible. Exiting.%s\n\n" "$err" "$norm"
-  writePlainTextLog "'curl' must be installed and accessible" "fatal" "fail" "preexec" 2
+  ([ "$logToConsole" -eq 0 ]) && writeLog "'curl' must be installed and accessible" "fatal" "fail" "preexec" 2
   exit 2
 fi
 if ! command -v jq >/dev/null; then
-  printf "\n%sThis script requires 'jq' be installed and accessible. Exiting.%s\n\n" "$err" "$norm"
+  printf "\n%sThis script requires that 'jq' is installed and accessible. Exiting.%s\n\n" "$err" "$norm"
-  writePlainTextLog "'jq' must be installed and accessible" "fatal" "fail" "preexec" 2
+  ([ "$logToConsole" -eq 0 ]) && writeLog "'jq' must be installed and accessible" "fatal" "fail" "preexec" 2
   exit 2
 fi
 if [ -z "$dnsRecords" ]; then
   badParam errMsg "You must specify at least one DNS record to update. Exiting."
-  writePlainTextLog "At least one DNS record to update must be specified" "fatal" "fail" "params" 1
 fi
 # verify the credentials file exists and is not empty (default check)
 if [ ! -f "$accountFile" ] || [ ! -s "$accountFile" ]; then
-  badParam errMsg "Cannot find the Cloudflare credentials file (${accountFile}). Exiting."
+  badParam errMsg "Cannot find the Cloudflare credentials file (${accountFile}) or it is empty. Exiting."
-  writePlainTextLog "Cannot find the specified Cloudflare credentials file (${accountFile})" "fatal" "fail" "params" 1
 fi
 if [ "$logToJournal" -eq 1 ] && ! command -v logger >/dev/null 2>&1; then
-  printf "\n%sThis script requires 'logger' be installed to write entries to your journaling system. Exiting.%s\n\n" "$err" "$norm"
+  printf "\n%sThis script requires that 'logger' is installed to write journald entries. Exiting.%s\n\n" "$err" "$norm"
   exit 2
 fi
-# turn off log file colourization if parameter is set
-if [ "$colourizeLogFile" -eq 0 ]; then
-  bold=""
-  cyan=""
-  err=""
-  magenta=""
-  norm=""
-  ok=""
-  warn=""
-  yellow=""
-fi
 
 ### initial log entries
-writePlainTextLog "starting '${scriptName}'" "info" "ok" "script"
+writeLog "starting '${scriptName}'" "info" "ok" "script"
-writePlainTextLog "script path: ${scriptPath}/${scriptName}" "debug" "ok" "startup"
+writeLog "script path: ${scriptPath}/${scriptName}" "debug" "ok" "startup"
-writePlainTextLog "credentials file: ${accountFile}" "debug" "ok" "startup"
+writeLog "credentials file: ${accountFile}" "debug" "ok" "startup"
 
 if [ "$ip4" -eq 1 ]; then
-  writePlainTextLog "mode: IPv4" "debug" "ok" "startup"
+  writeLog "mode: IPv4" "debug" "ok" "startup"
 elif [ "$ip6" -eq 1 ]; then
-  writePlainTextLog "mode: IPv6" "debug" "ok" "startup"
+  writeLog "mode: IPv6" "debug" "ok" "startup"
 fi
 
 # detect and report IP address
@@ -380,21 +436,21 @@ if [ -z "$ipAddress" ]; then
   # detect public ip address
   if [ "$ip4" -eq 1 ]; then
     if ! ipAddress="$(curl -s $ip4DetectionSvc)"; then
-      writePlainTextLog \
+      writeLog \
         "Unable to auto-detect this machine's public IP address; try again later or supply the IP address to be used" "err" "fail" "detectip" 10
       exit 10
     fi
   fi
   if [ "$ip6" -eq 1 ]; then
     if ! ipAddress="$(curl -s $ip6DetectionSvc)"; then
-      writePlainTextLog \
+      writeLog \
         "Unable to auto-detect this machine's public IP address; try again later or supply the IP address to be used" "err" "fail" "detectip" 10
       exit 10
     fi
   fi
-  writePlainTextLog "DDNS IP address (detected): $ipAddress" "info" "ok" "startup"
+  writeLog "DDNS IP address (detected): $ipAddress" "info" "ok" "startup"
 else
-  writePlainTextLog "DDNS IP address (supplied): $ipAddress" "info" "ok" "startup"
+  writeLog "DDNS IP address (supplied): $ipAddress" "info" "ok" "startup"
 fi
 
 # iterate DNS records to update
@@ -406,7 +462,7 @@ while [ -n "$dnsRecordsToUpdate" ] && [ "$dnsRecordsToUpdate" != "$dnsSeparator"
   if [ -z "$record" ]; then
     continue
   fi
-  writePlainTextLog "DNS host record '${record}' queued for update" "info" "ok" "startup"
+  writeLog "DNS host record '${record}' queued for update" "info" "ok" "startup"
 done
 
 ### read Cloudflare credentials
@@ -423,14 +479,14 @@ case "$accountFile" in
   ;;
 esac
 if [ -z "$cfKey" ]; then
-  writePlainTextLog "Cloudflare authorized API key (cfKey) is either null or undefined; please check your Cloudflare credentials file" "err" "fail" "creds" 21
+  writeLog "Cloudflare authorized API key (cfKey) is either null or undefined; please check your Cloudflare credentials file" "err" "fail" "creds" 21
   exit 21
 fi
 if [ -z "$cfZoneId" ]; then
-  writePlainTextLog "Cloudflare zone id (cfZoneId) is either null or undefined; please check your Cloudflare credentials file" "err" "fail" "creds" 22
+  writeLog "Cloudflare zone id (cfZoneId) is either null or undefined; please check your Cloudflare credentials file" "err" "fail" "creds" 22
   exit 22
 fi
-writePlainTextLog "Cloudflare credentials file read successfully" "debug" "ok" "creds"
+writeLog "Cloudflare credentials file read successfully" "debug" "ok" "creds"
 
 ### connect to Cloudflare and do what needs to be done!
 dnsRecordsToUpdate="$dnsRecords$dnsSeparator"
@@ -455,7 +511,7 @@ while [ -n "$dnsRecordsToUpdate" ] && [ "$dnsRecordsToUpdate" != "$dnsSeparator"
       -H "Authorization: Bearer ${cfKey}" \
       -H "Content-Type: application/json"
   )"; then
-    writePlainTextLog "Unable to connect to Cloudflare servers; please try again later." "err" "fail" "cflogin" 3
+    writeLog "Unable to connect to Cloudflare servers; please try again later." "err" "fail" "cflogin" 3
     exit 3
   fi
 
@@ -464,7 +520,7 @@ while [ -n "$dnsRecordsToUpdate" ] && [ "$dnsRecordsToUpdate" != "$dnsSeparator"
   cfSuccess="$(printf "%s" "$cfLookup" | jq -r '.success')"
   if [ "$cfSuccess" = "false" ]; then
     listCFErrors "$cfLookup" "cflogin"
-    writePlainTextLog "Cloudflare API error; review any previously logged 'CF-ERR:' lines for details." "err" "fail" "cflogin" 25
+    writeLog "Cloudflare API error; review any previously logged 'CF-ERR:' lines for details." "err" "fail" "cflogin" 25
     exit 25
   fi
 
@@ -472,18 +528,18 @@ while [ -n "$dnsRecordsToUpdate" ] && [ "$dnsRecordsToUpdate" != "$dnsSeparator"
 
   # skip to the next host if an existing host record cannot be found (this script *updates* only, does not create!)
   if [ "$resultCount" = "0" ]; then
-    writePlainTextLog "Cannot find an existing record matching '${record}' to update" "warn" "warn" "ddns"
+    writeLog "Cannot find an existing record matching '${record}' to update" "warn" "warn" "ddns"
     invalidDomainCount=$((invalidDomainCount + 1))
     continue
   fi
 
   objectId=$(printf "%s" "$cfLookup" | jq -r '.result | .[] | .id')
   currentIpAddr=$(printf "%s" "$cfLookup" | jq -r '.result | .[] | .content')
-  writePlainTextLog "The current IP address for '${record}' is ${currentIpAddr}" "debug" "ok" "ddns"
+  writeLog "The current IP address for '${record}' is ${currentIpAddr}" "debug" "ok" "ddns"
 
   # skip to next hostname if record already up to date
   if [ "$currentIpAddr" = "$ipAddress" ]; then
-    writePlainTextLog "The IP address for '${record}' is already up to date" "info" "ok" "ddns"
+    writeLog "The IP address for '${record}' is already up to date" "info" "ok" "ddns"
     continue
   fi
 
@@ -497,17 +553,17 @@ while [ -n "$dnsRecordsToUpdate" ] && [ "$dnsRecordsToUpdate" != "$dnsSeparator"
       -H "Content-Type: application/json" \
       --data "${updateJSON}"
   )"; then
-    writePlainTextLog "Unable to connect to Cloudflare servers; please try again later." "err" "fail" "ddns" 3
+    writeLog "Unable to connect to Cloudflare servers; please try again later." "err" "fail" "ddns" 3
     exit 3
   fi
 
   # note update success or failure
   cfSuccess="$(printf "%s" "$cfResult" | jq '.success')"
   if [ "$cfSuccess" = "true" ]; then
-    writePlainTextLog "The IP address for '${record}' successfully updated" "info" "ok" "ddns"
+    writeLog "The IP address for '${record}' successfully updated" "info" "ok" "ddns"
   else
     listCFErrors "$cfResult" "ddns"
-    writePlainTextLog "Unable to update the IP address for '${record}'" "err" "fail" "ddns"
+    writeLog "Unable to update the IP address for '${record}'" "err" "fail" "ddns"
     # do not exit with error, API error here is probably an update issue specific to this host
     # increment counter and note it after all processing finished
     failedHostCount=$((failedHostCount + 1))
@@ -516,14 +572,14 @@ done
 
 # exit
 if [ "$invalidDomainCount" -ne 0 ]; then
-  writePlainTextLog "${invalidDomainCount} invalid host(s) were supplied for updating" "warn" "warn" "ddns"
+  writeLog "${invalidDomainCount} invalid host(s) were supplied for updating" "warn" "warn" "ddns"
 fi
 if [ "$failedHostCount" -ne 0 ]; then
-  writePlainTextLog \
+  writeLog \
     "${failedHostCount} host update(s) failed; review 'CF-ERR:' lines in this log to help determine what may have gone wrong" "err" "fail" "ddns" 26
   exit 26
 else
-  writePlainTextLog "${scriptName} completed successfully" "info" "ok" "script"
+  writeLog "${scriptName} completed successfully" "info" "ok" "script"
 fi
 
 ### exit return codes