refactor(log): Use CLEF for JSON logs

Use Compact Log Event Format (CLEF) for JSON logs along with custom properties.

.gitattributes

@@ -55,6 +55,23 @@
 *.bat      text eol=crlf
 *.cmd      text eol=crlf
 
+# web frontend stack -- force LF so SRI hashes are always correct
+*.html     text eol=lf
+*.htm      text eol=lf
+*.css      text eol=lf
+*.min.css  text eol=lf
+*.js       text eol=lf
+*.min.js   text eol=lf
+*.php      text eol=lf
+
+# Visual Studio projects (Rider also)
+*.cs     diff=csharp
+*.sln    merge=union
+*.csproj merge=union
+*.vbproj merge=union
+*.fsproj merge=union
+*.dbproj merge=union
+
 # Serialisation
 *.json     text
 *.toml     text
@@ -76,6 +93,8 @@
 # Exclude files from exporting
 #
 
-.gitattributes export-ignore
+.gitattributes  export-ignore
-.gitignore     export-ignore
+.gitignore      export-ignore
-.gitkeep       export-ignore
+.gitkeep        export-ignore
+.idea           export-ignore
+.vscode         export-ignore

.gitignore

@@ -1 +1,35 @@
+#
+# JetBrains exclusions
+#
+
+riderModule.iml
+/_ReSharper.Caches/
+
+# User-specific stuff
+.idea/**/workspace.xml
+.idea/**/tasks.xml
+.idea/**/usage.statistics.xml
+.idea/**/dictionaries
+.idea/**/shelf
+
+# Generated files
+.idea/**/contentModel.xml
+.idea/**/GitCommitMessageStorage.xml
+
+# Sensitive or high-churn files
+.idea/**/dataSources/
+.idea/**/dataSources.ids
+.idea/**/dataSources.local.xml
+.idea/**/sqlDataSources.xml
+.idea/**/dynamic.xml
+.idea/**/uiDesigner.xml
+.idea/**/dbnavigator.xml
+
+# modules
+.idea_modules/
+
+# Editor-based Rest Client
+.idea/httpRequests
+
+# project-specific exclusions
 *.log

.idea/developer-tools.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="DeveloperToolsToolWindowSettingsV1" lastSelectedContentNodeId="base64-encoder-decoder" pluginVersion="9.0.0">
+    <developerToolsConfigurations />
+  </component>
+</project>

.idea/git_toolbox_prj.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="GitToolBoxProjectSettings">
+    <option name="commitMessageIssueKeyValidationOverride">
+      <BoolValueOverride>
+        <option name="enabled" value="true" />
+      </BoolValueOverride>
+    </option>
+    <option name="commitMessageValidationEnabledOverride">
+      <BoolValueOverride>
+        <option name="enabled" value="true" />
+      </BoolValueOverride>
+    </option>
+  </component>
+</project>

.idea/indexLayout.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="UserContentModel">
+    <attachedFolders />
+    <explicitIncludes />
+    <explicitExcludes />
+  </component>
+</project>

.idea/misc.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="com.itcodebox.notebooks.projectservice.ProjectUIState">
+    <option name="selectedNotebookId" value="1" />
+    <option name="selectedChapterId" value="1" />
+    <option name="selectedNoteId" value="1" />
+  </component>
+</project>

.idea/projectSettingsUpdater.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="RiderProjectSettingsUpdater">
+    <option name="singleClickDiffPreview" value="1" />
+    <option name="unhandledExceptionsIgnoreList" value="1" />
+    <option name="vcsConfiguration" value="3" />
+  </component>
+</project>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="" vcs="Git" />
+  </component>
+</project>

Folder.DotSettings

@@ -0,0 +1,4 @@
+<wpf:ResourceDictionary xml:space="preserve" xmlns:x="http://schemas.microsoft.com/winfx/2006/xaml" xmlns:s="clr-namespace:System;assembly=mscorlib" xmlns:ss="urn:shemas-jetbrains-com:settings-storage-xaml" xmlns:wpf="http://schemas.microsoft.com/winfx/2006/xaml/presentation">
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=cfddns/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=mydomain/@EntryIndexedValue">True</s:Boolean>
+	<s:Boolean x:Key="/Default/UserDictionary/Words/=myserver/@EntryIndexedValue">True</s:Boolean></wpf:ResourceDictionary>

README.md

@@ -7,7 +7,7 @@ Update your *existing* Cloudflare DNS records with your current (dynamic) IP add
 <!-- toc -->
 
 - [Prerequisites](#prerequisites)
-- [cfddns&#46;sh](#cfddns%2346sh)
+- [cfddns script](#cfddns-script)
   * [Installation](#installation)
   * [Usage](#usage)
     + [Parameters](#parameters)
@@ -15,10 +15,10 @@ Update your *existing* Cloudflare DNS records with your current (dynamic) IP add
   * [File structure](#file-structure)
   * [Bearer token](#bearer-token)
   * [Zone ID](#zone-id)
-- [cfddns.service](#cfddnsservice)
+- [cfddns systemd service unit](#cfddns-systemd-service-unit)
-  * [IP4 and/or IP6](#ip4-andor-ip6)
+  * [IP4 or IP6](#ip4-or-ip6)
     + [Examples](#examples)
-- [cfddns.timer](#cfddnstimer)
+- [cfddns systemd timer unit](#cfddns-systemd-timer-unit)
   * [Activation](#activation)
 - [Logging](#logging)
   * [Using Logwatch to monitor this script](#using-logwatch-to-monitor-this-script)
@@ -37,7 +37,7 @@ apt install -y curl jq
 
 While the script does *not* require root privileges, you will need sudo/root access to install the *systemd* service and timer.
 
-## cfddns&#46;sh
+## cfddns script
 
 ### Installation
 
@@ -56,7 +56,7 @@ I recommend putting this script in your */usr/local/bin* directory or somewhere
    sudo chmod +x /usr/local/bin/cfddns.sh
    ```
 
-> Note: You can rename *cfddns.sh* to anything you want, the script will auto-update itself. However, you **must** manually update the systemd service file (*cfddns.service*) `ExecStart` line as [explained below](#cfddns.service).
+> Note: You can rename *cfddns.sh* to anything you want, the script will auto-update itself. However, you **must** manually update the systemd service file (*cfddns.service*) `ExecStart` line as [explained below](#cfddns-script).
 
 ### Usage
 
@@ -111,6 +111,13 @@ cfZoneId=83d564234134513245311b23412331dd
 
 You can save the file as anything you like and anywhere you’d like as long as you inform the script of its location using the `--credentials` parameter. By default, the script will look for a file named *cloudflare.credentials* in the same path as the script.
 
+Please remember that this file basically contains a password! As a result, it should be protected and access limited to the root account:
+
+```bash
+chown root:root /path/to/cloudflare.credentials
+chmod 600 /path/to/cloudflare.credentials
+```
+
 ### Bearer token
 
 I chose to use an API bearer token instead of a username/password or Global API token for security reasons. Your username/password and Global API token provide unfettered access to your account so if anyone gets hold of them, they can do anything to your account. An API bearer token, by contrast, can only do what you authorize it to do and you can revoke it at any time. Therefore, I suggest making a bearer token that is based on the “Edit zone DNS” template and restricted to the specific domain/zone you wish to update. Cloudflare provides an [excellent article](https://support.cloudflare.com/hc/en-us/articles/200167836-Managing-API-Tokens-and-Keys) on how to generate this token.
@@ -124,7 +131,7 @@ This is required by the Cloudflare API so it knows which zone you are editing an
 To get your Zone ID, log into your Cloudflare account and open the domain in question. On the overview page, scroll down a bit and look to the right. You will see your Zone ID listed there. Copy that string into your configuration file.
 
 
-## cfddns.service
+## cfddns systemd service unit
 
 This file **must** be copied to your */etc/systemd/system* directory (or equivalent directory if you're not running Debian/Ubuntu). If you change the name of the cfddns&#46;sh file, you must update the filename in the `ExecStart` line as shown below:
 
@@ -136,13 +143,7 @@ ExecStart=/full/path/to/your/renamed.file -parameter1 -parameter2 -parameter...
 ...
 ````
 
-Don’t forget to reload systemd after copying this file so it is recognized by the system! On most systems you can do this by running the following as root or via sudo:
+### IP4 or IP6
-
-```bash
-systemctl daemon-reload
-```
-
-### IP4 and/or IP6
 
 The cfddns.service file includes two *ExecStart* lines, one without a specified IP-protocol parameter (default IP4) and the other with the -6 (IP6) parameter. The service will run the cfddns&#46;sh script in default (IP4) mode with specified parameters first and then will run the script again in IP6 mode with specified parameters.
 
@@ -183,9 +184,9 @@ The cfddns.service file includes two *ExecStart* lines, one without a specified
    ...
    ```
 
-## cfddns.timer
+## cfddns systemd timer unit
 
-This is the timer file that tells your system how often to call the *cfddns.service* file which runs the *cfddns&#46;sh* script.  By default, the timer is set for 5 minutes after the system boots up (to allow for other processes to initialize even on slower systems like a RasPi) and is then run every 15 minutes thereafter.  Remember when setting your timer that Cloudflare limits API calls to 1200 every 5 minutes.
+Just like the service file unit, this file **must** be copied to your */etc/systemd/system* directory (or equivalent directory if you're not running Debian/Ubuntu). This timer file unit tells your system how often to call the *cfddns.service* file which runs the *cfddns&#46;sh* script.  By default, the timer is set for 5 minutes after the system boots up (to allow for other processes to initialize even on slower systems like a RasPi) and is then run every 15 minutes thereafter.  Remember when setting your timer that Cloudflare limits API calls to 1200 every 5 minutes.
 
 You can change the timer by modifying the relevant section of the *cfddns.timer* file:
 
@@ -198,6 +199,12 @@ OnUnitActiveSec=15min
 *OnBootSec* is how long to wait after the system boots up before executing the *cfddns.service*.  *OnUnitActiveSec* will then wait the specified time from that first (after boot) call or after the timer is explicitly started before calling *cfddns.service* again.  I recommend setting OnUnitActiveSec to a low value (like 2 minutes) for testing then setting it to a more reasonable time (like 15
 minutes) after everything is working.
 
+After you’ve copied both the systemd unit and this timer unit, don’t forget to reload the systemd daemon so they are recognized by the system! On most systems you can do this by running the following as root or via sudo:
+
+```bash
+systemctl daemon-reload
+```
+
 ### Activation
 
 You can start the timer system immediately via *systemctl*
@@ -220,6 +227,8 @@ systemctl status cfddns.timer
 
 It is NOT necessary to enable/start the *cfddns.service*, only the timer needs to be active.
 
+Also remember that if you make changes to settings like `OnUnitActiveSec` while testing or after testing is complete you *must* reload the systemd daemon! It will restart the appropriate units for you and your new settings will take effect immediately.
+
 ## Logging
 
 The script logs every major action it takes and provides details on any errors it encounters in the log file (see the [parameters section](#parameters) for details about setting log location and name).  If errors are encountered, they are colour coded red and an explanation of the error code is provided.

etc/logwatch/README.md

@@ -15,17 +15,17 @@ If you need help getting logwatch installed and set-up, please [check out my blo
 
 <!-- toc -->
 
-- [LogFile Group file (/etc/logwatch/conf/logfiles/cfddns.conf)](#logfile-group-file-etclogwatchconflogfilescfddnsconf)
+- [LogFile Group file](#logfile-group-file)
   * [Log file location](#log-file-location)
   * [Archive location and name format](#archive-location-and-name-format)
   * [External script for timestamp processing](#external-script-for-timestamp-processing)
-- [Service definition file (/etc/logwatch/conf/services/cfddns.conf)](#service-definition-file-etclogwatchconfservicescfddnsconf)
+- [Service definition file](#service-definition-file)
   * [LogFile Group file definition](#logfile-group-file-definition)
   * [Report title](#report-title)
   * [Detail level](#detail-level)
-- [Service script (/etc/logwatch/scripts/services/cfddns)](#service-script-etclogwatchscriptsservicescfddns)
+- [Service script](#service-script)
   * [Detail levels](#detail-levels)
-- [Timestamp processing script (/etc/logwatch/scripts/shared/sqfullstampanywhere)](#timestamp-processing-script-etclogwatchscriptssharedsqfullstampanywhere)
+- [Timestamp processing script](#timestamp-processing-script)
   * [The time format specification](#the-time-format-specification)
   * [The search REGEX](#the-search-regex)
 - [Testing](#testing)
@@ -33,7 +33,9 @@ If you need help getting logwatch installed and set-up, please [check out my blo
 
 <!-- tocstop -->
 
-## LogFile Group file (/etc/logwatch/conf/logfiles/cfddns.conf)
+## LogFile Group file
+
+> This file is located within the repo at */etc/logwatch/conf/logfiles/cfddns.conf*
 
 ### Log file location
 
@@ -79,7 +81,9 @@ The script file is called with an asterisk (*\**) before the filename.
 
 If you change the name of this file, you will have to change this line. Remember that whatever you type here as a name is converted to all-lowercase so your filename should be all lowercase also.
 
-## Service definition file (/etc/logwatch/conf/services/cfddns.conf)
+## Service definition file
+
+> This file is located within the repo at */etc/logwatch/conf/services/cfddns.conf*
 
 ### LogFile Group file definition
 
@@ -119,7 +123,9 @@ Simply change it to the value you want enforced.  For example, here I'm setting
 Detail = 5
 ```
 
-## Service script (/etc/logwatch/scripts/services/cfddns)
+## Service script
+
+> This file is located within the repo at */etc/logwatch/scripts/services/cfddns*
 
 Logwatch calls any script with a name that **matches the service name**.  You'll notice that I just named everything *cfddns* to keep things simple.  You can change this to whatever you want.  If you changed the service name to *"cloudflare*.conf", for example, you would have to rename this script file to "*cloudflare*" with no extension.  Note: The script is a PERL file (note the
 shebang) but it can be written in any language.
@@ -160,7 +166,9 @@ The script supports four (4) detail levels as follows:
   - **Use this detail level only when you need to see the entire log file and cannot otherwise access the log file.**
   - Depending on how your logwatch treats this log dump, you may see gibberish control codes like *\e[0m;]*. If this is the case, run the script with the `--no-colour` or `--nc` option to remove ANSI colour formatting.
 
-## Timestamp processing script (/etc/logwatch/scripts/shared/sqfullstampanywhere)
+## Timestamp processing script
+
+> This file is located within the repo at */etc/logwatch/scripts/shared/sqfullstampanywhere*
 
 This is basically a modified version of the '*applyeurodate*' script that comes with Logwatch.  It had to be modified to search within [square brackets] and to accept characters coming before the stamp (i.e. ANSI colour codes).  If you change the '**stamp**' variable in the updater script to update the timestamp to your liking (which to totally fine!) then you'll probably have to update this file.  There are two lines you need to modify to suit your new '**stamp**' variable.
 

logger.sh

@@ -0,0 +1,161 @@
+#!/bin/sh
+
+getTimeStamp() {
+  (date -u +"%Y-%m-%dT%H:%M:%SZ")
+}
+
+getHostname() {
+  (hostname -s)
+}
+
+lowercase() {
+  echo "$1" | tr '[:upper:]' '[:lower:]'
+}
+
+uppercase() {
+  echo "$1" | tr '[:lower:]' '[:upper:]'
+}
+
+standardizeLogLevels() {
+  LEVEL_TO_STANDARDIZE="$(lowercase "$1")"
+  case "$LEVEL_TO_STANDARDIZE" in
+  "critical" | "crit" | "fatal")
+    echo "CRIT"
+    ;;
+  "error" | "err")
+    echo "ERR"
+    ;;
+  "warning" | "warn")
+    echo "WARN"
+    ;;
+  "information" | "info")
+    echo "INFO"
+    ;;
+  "debug" | "verbose")
+    echo "DEBUG"
+    ;;
+  *)
+    echo "INFO"
+    ;;
+  esac
+}
+
+translateLogLevelsToJournalD() {
+  LEVEL_TO_TRANSLATE="$(standardizeLogLevels "$1")"
+  case "$LEVEL_TO_TRANSLATE" in
+  "CRIT")
+    echo 2
+    ;;
+  "ERR")
+    echo 3
+    ;;
+  "WARN")
+    echo 4
+    ;;
+  "INFO")
+    echo 6
+    ;;
+  "DEBUG")
+    echo 7
+    ;;
+  *)
+    echo 6
+  esac
+}
+
+# modified syslog fmt: <ISO-8601 timestamp> <hostname> <tag>[pid]: [LEVEL:$2] [STATUS:$3] <MESSAGE:$1> (<OPERATION:$4>:<CODE:$5>)
+writePlainTextLog() {
+  # not enough information to generate a meaningful log message
+  if [ -z "$1" ] || [ $# -lt 3 ]; then
+    return
+  fi
+
+  printf "%s %s %s[%s]: [%s] [%s] %s (%s:%s)\n" \
+    "$(getTimeStamp)" "$(getHostname)" "$LOG_PROGRAM_NAME" "$$" \
+    "$(standardizeLogLevels "$2")" "$(uppercase "$3")" "$1" "${4:-UNSPECIFIED}" "${5:-999}"
+}
+
+# $1: message, $2: level, $3: status, [$4: operation], [$5: code]
+writeJsonLog() {
+  # not enough information to generate a meaningful log message
+  if [ -z "$1" ] || [ $# -lt 3 ]; then
+    return
+  fi
+
+  PROPS="$(
+    jq --null-input \
+      --arg timestamp "$(getTimeStamp)" \
+      --arg hostname "$(getHostname)" \
+      --arg pName "$LOG_PROGRAM_NAME" \
+      --arg pid "$$" \
+      --arg level "$(standardizeLogLevels "$2")" \
+      --arg status "$(uppercase "$3")" \
+      --arg message "$1" \
+      --arg operation "${4:-UNSPECIFIED}" \
+      --arg code "${5:-999}" \
+      '{"timestamp":$timestamp, "hostname":$hostname, "programName":$pName, "pid":$pid, "level":$level, "status":$status, "message":$message, "operation":$operation, "code":$code}' \
+  )"
+
+  echo "$PROPS" | jq "."
+  return
+}
+
+# $1: message, $2: level, $3: status, $4: operation, $5: code
+writeJournalDLog() {
+  # not enough information to generate a meaningful log message
+  if [ -z "$1" ] || [ $# -lt 3 ]; then
+    return
+  fi
+
+  J_TIMESTAMP="$(getTimeStamp)"
+  J_PID="$$"
+  J_PRIO="$(translateLogLevelsToJournalD "$2")"
+  J_STAT="$(uppercase "$3")"
+
+  logger --journald <<end
+SYSLOG_TIMESTAMP=${J_TIMESTAMP}
+SYSLOG_FACILITY=3
+SYSLOG_IDENTIFIER=${LOG_PROGRAM_NAME}
+SYSLOG_PID=${J_PID}
+PRIORITY=${J_PRIO}
+STATUS=${J_STAT}
+MESSAGE=$1
+OPERATION=${4:-UNSPECIFIED}
+ERRNO=${5:-999}
+end
+
+  return
+}
+
+LOG_PROGRAM_NAME="cfddns"
+
+if [ "$1" = "text" ]; then
+  writePlainTextLog # this will do nothing
+  writePlainTextLog "Testing plain-text logging output" # this will also do nothing
+  writePlainTextLog "Informational plain-text log message" "information" "ok" # info message
+  writePlainTextLog "Plain-text message forced to info level" "whoops" "ok" # level translated to default
+  writePlainTextLog "Warning! This is plain-text." "warning" "flagged" # warning level with 'flagged' status
+  writePlainTextLog "Error sample! This is plain-text." "err" "error" # error level message
+  writePlainTextLog "Unable to continue, critical failure. (This is plain-text)" "fatal" "exit" "startup" "2" # critical level error with operation and error code
+elif [ "$1" = "json" ]; then
+  writeJsonLog # this will do nothing
+  writeJsonLog "This is a test message" # this will also do nothing
+  writeJsonLog "Informational JSON log message" "information" "ok" # info message
+  writeJsonLog "JSON message forced to info level" "whoops" "ok" # level translated to default
+  writeJsonLog "Warning! This is JSON." "warning" "flagged" # warning level with 'flagged' status
+  writeJsonLog "Error sample! This is JSON." "err" "error" # error level message
+  writeJsonLog "Unable to continue, critical failure. (This is JSON)" "fatal" "exit" "startup" "2" # critical level error with operation and error code
+elif [ "$1" = "journal" ]; then
+  writeJournalDLog # this will do nothing
+  writeJournalDLog "This is a test message" # this will also do nothing
+  writeJournalDLog "Informational JOURNALD log message" "information" "ok" # info message
+  writeJournalDLog "JOURNALD message forced to info level" "whoops" "ok" # level translated to default
+  writeJournalDLog "Warning! This is a JOURNALD message." "warning" "flagged" # warning level with 'flagged' status
+  writeJournalDLog "Error sample! This is a JOURNALD." "err" "error" # error level message
+  writeJournalDLog "Unable to continue, critical failure. (This is JOURNALD message)" "fatal" "exit" "startup" "2" # critical level error with operation and error code
+else
+  printf "\nPlease specify either 'text', 'json', or 'journal' after the script name. Exiting.\n\n"
+  exit 1
+fi
+
+exit 0