← back to Wallco Ai
security-monitor.sh: add cron + service-unit drift checks
c0ef0e824eeef427dfdf9f96b38c633336427ead · 2026-05-30 12:15:14 -0700 · Steve Abrams
amco-docker-forensics found the May 28 attacker's persistence was a
docker-start cron in /var/spool/cron/crontabs/root — the original 4
checks missed it entirely.
Added two checks:
- check_cron_files() — sha256-tree over /var/spool/cron/crontabs/* +
/etc/cron.d/* + /etc/cron.{daily,hourly,weekly,monthly}/*. Content
hashes so in-place edits to a known file (the attack shape) are
caught, not just new filenames.
- check_services() — sha256-tree over /etc/systemd/system/*.service.
Same edit-detection benefit; service-unit persistence is more
common than timer-unit.
Both bootstrap to /root/scripts/known-good-{cron,services}.snapshot
on first run (INFO, not CRITICAL). Snapshot-update one-liners
embedded in the alert body so a legit admin change is one paste.
bash -n clean. Tested DRY_RUN locally + live run on prod — 2 new
snapshots written, second run idempotent (0 new alerts). Cron entry
at */10 untouched.
Files touched
M scripts/security-monitor.sh
Diff
commit c0ef0e824eeef427dfdf9f96b38c633336427ead
Author: Steve Abrams <steve@designerwallcoverings.com>
Date: Sat May 30 12:15:14 2026 -0700
security-monitor.sh: add cron + service-unit drift checks
amco-docker-forensics found the May 28 attacker's persistence was a
docker-start cron in /var/spool/cron/crontabs/root — the original 4
checks missed it entirely.
Added two checks:
- check_cron_files() — sha256-tree over /var/spool/cron/crontabs/* +
/etc/cron.d/* + /etc/cron.{daily,hourly,weekly,monthly}/*. Content
hashes so in-place edits to a known file (the attack shape) are
caught, not just new filenames.
- check_services() — sha256-tree over /etc/systemd/system/*.service.
Same edit-detection benefit; service-unit persistence is more
common than timer-unit.
Both bootstrap to /root/scripts/known-good-{cron,services}.snapshot
on first run (INFO, not CRITICAL). Snapshot-update one-liners
embedded in the alert body so a legit admin change is one paste.
bash -n clean. Tested DRY_RUN locally + live run on prod — 2 new
snapshots written, second run idempotent (0 new alerts). Cron entry
at */10 untouched.
---
scripts/security-monitor.sh | 107 +++++++++++++++++++++++++++++++++++++++++++-
1 file changed, 106 insertions(+), 1 deletion(-)
diff --git a/scripts/security-monitor.sh b/scripts/security-monitor.sh
index b930cf0..8c73c7a 100755
--- a/scripts/security-monitor.sh
+++ b/scripts/security-monitor.sh
@@ -35,6 +35,15 @@ PASSWD_SNAP="${SCRIPT_DIR}/known-good-passwd.snapshot"
SUDOERSD_SNAP="${SCRIPT_DIR}/known-good-sudoersd.snapshot"
AUTHKEYS_SNAP="${SCRIPT_DIR}/known-good-authkeys.snapshot"
TIMERS_SNAP="${SCRIPT_DIR}/known-good-timers.snapshot"
+# 2026-05-30 hardening — added after amco-docker-forensics found attacker
+# persistence in /var/spool/cron/crontabs/root (a docker-start cron) that the
+# original 4 checks missed entirely. Also widens the systemd watch to .service
+# units (not just .timer) since service-unit persistence is the more common
+# vector. Both new checks use sha256 over the file TREE so that in-place edits
+# to a known file (the docker-start attack shape) are caught, not just new
+# filenames appearing.
+CRON_SNAP="${SCRIPT_DIR}/known-good-cron.snapshot"
+SERVICES_SNAP="${SCRIPT_DIR}/known-good-services.snapshot"
ts() { date '+%Y-%m-%d %H:%M:%S %Z'; }
@@ -154,7 +163,101 @@ check_timers() {
fi
}
-# ---- Check 5: useradd events in journalctl over last 15 min ----
+# ---- helper: per-file sha256 snapshot over a list of paths ----
+# Outputs "<sha256> <path>" lines, sorted by path. Missing files are skipped
+# (find -print0 handles that already). Returns the empty string if no files
+# exist in the watched tree — that's a valid baseline (clean system).
+hash_tree() {
+ # $@ = list of dirs to walk (files at depth 1, NOT recursive on dir contents
+ # beyond depth 1 — cron.d/* etc. are flat dirs)
+ local d
+ for d in "$@"; do
+ if [ -d "$d" ]; then
+ find "$d" -maxdepth 1 -type f -print0 2>/dev/null
+ fi
+ done | sort -z | xargs -0 -r sha256sum 2>/dev/null
+}
+
+# ---- Check 5: cron persistence ----
+# Covers /var/spool/cron/crontabs/* (per-user crontabs, INCLUDING root — the
+# vector the amco-docker-forensics attacker used) + /etc/cron.d/* + the four
+# /etc/cron.{daily,hourly,weekly,monthly}/ drop-in dirs. Uses content hashes
+# so an edit to /var/spool/cron/crontabs/root (vs a new file) is also caught.
+check_cron_files() {
+ local dirs=(
+ /var/spool/cron/crontabs
+ /etc/cron.d
+ /etc/cron.daily
+ /etc/cron.hourly
+ /etc/cron.weekly
+ /etc/cron.monthly
+ )
+ local current
+ current=$(hash_tree "${dirs[@]}")
+ ensure_snapshot "$CRON_SNAP" "echo \"$current\""
+ local baseline
+ baseline=$(cat "$CRON_SNAP")
+ if [ "$current" != "$baseline" ]; then
+ # diff hash lines to show added/changed/removed
+ local added removed
+ added=$(comm -23 <(echo "$current" | sort) <(echo "$baseline" | sort))
+ removed=$(comm -13 <(echo "$current" | sort) <(echo "$baseline" | sort))
+ local detail=""
+ if [ -n "$added" ]; then
+ detail+="ADDED or MODIFIED (sha256 file):\n${added}\n\n"
+ # show first 50 lines of each newly-hashed file path for context
+ local path
+ while read -r _ path; do
+ [ -z "$path" ] && continue
+ detail+="--- ${path} ---\n"
+ detail+="$(cat "$path" 2>/dev/null | head -50)\n\n"
+ done <<< "$added"
+ fi
+ if [ -n "$removed" ]; then
+ detail+="REMOVED (was in baseline, now gone):\n${removed}\n\n"
+ fi
+ log_alert "CRITICAL" "cron file drift (cron.d/spool/cron.{daily,hourly,weekly,monthly})" \
+ "Cron persistence drift on ${HOSTNAME_SHORT}. This is the persistence vector the May 28 docker-start cron used (placed in /var/spool/cron/crontabs/root).\n\n${detail}\nIf intentional, update snapshot:\n bash -c 'for d in /var/spool/cron/crontabs /etc/cron.d /etc/cron.daily /etc/cron.hourly /etc/cron.weekly /etc/cron.monthly; do [ -d \"\$d\" ] && find \"\$d\" -maxdepth 1 -type f -print0; done | sort -z | xargs -0 -r sha256sum' > ${CRON_SNAP}"
+ fi
+}
+
+# ---- Check 6: new/modified systemd .service units ----
+# Same content-hash pattern as cron — catches both new files AND edits to
+# existing service units (a common backdoor: append ExecStartPost= to a unit
+# that's already trusted by the admin). Watches /etc/systemd/system/*.service
+# directly (NOT the package-managed /lib/systemd/system tree — admin overrides
+# live in /etc/).
+check_services() {
+ local current
+ # only files matching *.service at the top level — keep parity with timers check
+ current=$(find /etc/systemd/system -maxdepth 1 -type f -name '*.service' -print0 2>/dev/null \
+ | sort -z | xargs -0 -r sha256sum 2>/dev/null)
+ ensure_snapshot "$SERVICES_SNAP" "echo \"$current\""
+ local baseline
+ baseline=$(cat "$SERVICES_SNAP")
+ if [ "$current" != "$baseline" ]; then
+ local added removed
+ added=$(comm -23 <(echo "$current" | sort) <(echo "$baseline" | sort))
+ removed=$(comm -13 <(echo "$current" | sort) <(echo "$baseline" | sort))
+ local detail=""
+ if [ -n "$added" ]; then
+ detail+="ADDED or MODIFIED:\n${added}\n\n"
+ local path
+ while read -r _ path; do
+ [ -z "$path" ] && continue
+ detail+="--- ${path} ---\n"
+ detail+="$(cat "$path" 2>/dev/null | head -40)\n\n"
+ done <<< "$added"
+ fi
+ if [ -n "$removed" ]; then
+ detail+="REMOVED:\n${removed}\n\n"
+ fi
+ log_alert "CRITICAL" "/etc/systemd/system/*.service drift" \
+ "Service unit drift on ${HOSTNAME_SHORT}. Service-unit persistence is more common than timer-unit persistence; this catches both new units and in-place edits (e.g. an appended ExecStartPost=).\n\n${detail}\nIf intentional, update snapshot:\n find /etc/systemd/system -maxdepth 1 -type f -name '*.service' -print0 | sort -z | xargs -0 -r sha256sum > ${SERVICES_SNAP}"
+ fi
+}
+
+# ---- Check 7: useradd events in journalctl over last 15 min ----
check_useradd_journal() {
if ! command -v journalctl >/dev/null 2>&1; then return; fi
local events
@@ -182,6 +285,8 @@ main() {
check_sudoersd
check_authkeys
check_timers
+ check_cron_files
+ check_services
check_useradd_journal
if [ "$DRY_RUN" = "1" ]; then
← 1ac6616 cactus-curator: bulk Publish now animates published cards ou
·
back to Wallco Ai
·
Force-publish 190 clean drunk-animals from 5/29 to live (Ste 167dee0 →