#!/bin/bash
###############################################################################
# forensic-watch.sh — always-on black-box recorder                     v2.2
#
# WHY THIS EXISTS
#   The box goes fully unresponsive (no SSH, no services) for ~5 min and then
#   recovers. You therefore CANNOT log in and capture in-flight state — by the
#   time you have a shell, the evidence is gone. This daemon samples cheap
#   /proc counters to disk continuously, so the next outage is recorded whether
#   or not anyone can log in.
#
#   Critically: if the OS hangs hard, this process stops being scheduled too.
#   That is itself the signal — the GAP in the sample file, plus the last
#   sample before it, tells us which failure mode fired:
#
#     last sample shows mem_avail collapsing + swap churn  -> memory exhaustion
#     last sample shows blocked(D-state) spiking, mem fine -> IO / storage stall
#     last sample shows steal% high, mem+io fine           -> hypervisor / noisy neighbour
#     no degradation at all, then a clean gap              -> host-level event, NOT us
#
# WHAT CHANGED IN v2  (2026-07-28, after the 09:38:38 capture)
#   v1 proved the allocator is php-cgi, not MariaDB: a 6.09 GB PHP burst across
#   69 processes appeared and vanished inside ~20 seconds, and 24 of the top 30
#   belonged to ONE cPanel account. v1 could not tell us what those processes
#   were serving, because:
#
#     1. It tailed only the ERP domlog. This is a shared cPanel box with ~20
#        accounts; the account that actually caused the burst was invisible.
#        -> v2 tails EVERY recently-written domlog and ranks vhosts by traffic.
#     2. "top 30 by RSS" truncated 39 of the 69 processes at the peak.
#        -> v2 dumps a per-user rollup over ALL processes, plus top 60.
#     3. Triggers were mem/load/blocked/swapout only. Those are LAGGING for a
#        20-second burst — the 07:56 dump fired on swapout after the event had
#        already ended and captured nothing but the aftermath.
#        -> v2 adds leading triggers: php_procs, php_rss, tcp_estab.
#     4. No run-up context in the dump; you had to go back to the TSV by hand.
#        -> v2 inlines the last 30 samples into every incident file.
#
#   The per-sample path is now ONE ps(1) call instead of five, so v2 is cheaper
#   than v1 despite recording three extra columns.
#
# WHAT CHANGED IN v2.1  (2026-07-30, after 8,319 samples on 07-29/07-30)
#   v2's leading triggers worked — they caught a 2,320 MB burst that v1's
#   thresholds would have slept through — but its vhost tables were unusable, so
#   the one remaining question (WHICH URL) still could not be answered:
#
#     1. Every row of the ranking read exactly "200". The count was non-static
#        lines within `tail -200`, which saturates for any log of 200+ lines. It
#        ranked log length, not traffic. -> now counted inside a real
#        DOMLOG_WIN_MIN-minute window by matching the Apache timestamp.
#     2. Each vhost appeared 2-4 times under different paths (cPanel keeps both
#        domlogs/<domain> and domlogs/<user>/<domain> for one file). This also
#        skewed the top-IP table unevenly. -> deduped on inode, keeping the
#        per-user path because it names the cPanel account.
#     3. '*-bytes_log' did not match 'kurri-imapbytes_log', so mail byte
#        counters were ranked as vhosts. -> widened to '*bytes_log', plus
#        .bkup/.localhost/proxy-subdomains noise.
#     4. "last 25 requests" never aggregated, so 40 one-off scanner paths looked
#        the same as 40 hits on one expensive endpoint — opposite diagnoses.
#        -> added top-request-path tables, global and per-vhost.
#     5. tcp_estab was a bad trigger: all 28 apache-spawn storms in the window
#        carried near-zero PHP (apache=153 at php_rss=350MB), because bots
#        hitting nonexistent .php are 404'd without starting an interpreter.
#        -> retired as a trigger (still recorded); php_rss 2000->1500; added
#        mysql_conn, which went 1->19 at the exact peak of the 21:51 burst.
#     6. The 07-29 22:18 event lost 2.9 GB of MemAvailable and recovered inside
#        30s with only 427 MB of PHP resident, at 18% iowait. That is page-cache
#        eviction, not anon pressure, and nothing recorded enough to prove it.
#        -> added vm tuning, PSI, Committed_AS/Cached/Dirty, and a scan for
#        backup/dump/AV processes.
#
#
# WHAT CHANGED IN v2.2  (2026-08-27, after 231 incident dumps on 08-12..08-27)
#   v2.1 fixed the vhost tables and they worked — but five more OOM kills of
#   mariadbd went by (08-12, 08-13, 08-14, 08-15, 08-25) and the last question
#   STILL could not be answered: which URL. The cause turned out to be
#   structural, not a threshold or a parsing bug:
#
#     1. Apache writes the access log only when a request COMPLETES. An
#        in-flight burst is therefore invisible to ANY domlog-derived table, no
#        matter how well ranked. On 08-03 the top account held 4 GB of php-cgi
#        while its domlog showed zero requests in the window.
#        -> added LIVE REQUEST ATTRIBUTION: read /proc/<pid>/environ of the
#           running workers. Under the EA4 cgi handler each request gets its own
#           interpreter and Apache hands it the CGI environment, so REQUEST_URI,
#           QUERY_STRING, SCRIPT_FILENAME, HTTP_HOST and REMOTE_ADDR are sitting
#           in the process the whole time. Aggregated first (40 hits on one
#           endpoint vs 40 one-off scanner paths are opposite diagnoses), then
#           listed longest-running-first with the full query string. Always run
#           for the burst user AND for 'evolution', so an ERP episode is
#           attributed even when another tenant is the heavier consumer.
#        NOTE: php-fpm children do not carry a per-request CGI env — for FPM
#           accounts this section says so and points at the fpm slow log.
#     2. "busiest 3 vhosts" ranks by completed requests, so the account actually
#        melting the box could sit at rank 9 and never be expanded.
#        -> the burst user's vhosts are now ALWAYS expanded, resolved from
#           /var/cpanel/users/<user> DNS entries.
#     3. RETAIN_DAYS 14 -> 30. The 08-12 11:55 ERP pile-up (137 workers, Apache
#        pinned at MaxRequestWorkers, box-wide HTTP outage) is the single most
#        interesting event we have recorded, and its dumps had already aged out
#        by the time the data was pulled on 08-27. 30 days of TSV+dumps is a few
#        hundred MB; losing the one event that matters costs another fortnight.
#     4. TWO watcher instances were found running concurrently: 17,190 rows/day
#        against 8,640 expected at a 10s interval, every timestamp duplicated.
#        The pidfile guard cannot stop this because the TERM trap removes the
#        file, so a restart race or @reboot-plus-manual-start walks straight
#        past it. -> added a flock(2) singleton, which the kernel holds for the
#        life of the process and which cannot go stale.
#        NB: data recorded 08-12..08-27 is doubled. Dedupe on the iso column
#        before counting anything: awk -F'\t' '!seen[$1]++'
# USAGE
#   nohup ./forensic-watch.sh > /dev/null 2>&1 &          # start (survives logout)
#   ./forensic-watch.sh -i 5 -d /var/log/erp-forensics    # 5s interval, custom dir
#   ./forensic-watch.sh --status                          # show today's peaks + gaps
#   ./forensic-watch.sh --gaps                            # list detected outages
#   ./forensic-watch.sh --stop
#
#   Survive reboot:  add to root crontab ->  @reboot /path/to/forensic-watch.sh -d /var/log/erp-forensics
#
# COST
#   ~1 syscall-cheap read of /proc plus one ps(1) per interval. MariaDB is only
#   queried every 6th sample (or immediately when a threshold trips), with a 2s
#   connect timeout, so the recorder cannot itself contribute to the problem.
#   Incident dumps are rate-limited to one per minute.
#
# Read-only. Never kills, restarts or reconfigures anything.
###############################################################################

INTERVAL=10
OUTDIR="$(pwd)/samples"
RETAIN_DAYS=30
ACTION="run"
PIDFILE=""

# incident thresholds — LAGGING (the box is already in trouble)
T_MEM_AVAIL_PCT=12      # MemAvailable below this % of MemTotal
T_LOAD_PER_CPU=8        # load1 / nproc above this
T_BLOCKED=10            # procs in uninterruptible (D) state
T_SWAPOUT=2000          # pages swapped out per second

# incident thresholds — LEADING (catch the burst while it is still running).
# Baseline on prod is php_procs 1-13, php_rss <1000 MB.
# The 09:38:38 event hit 69 procs / 6086 MB, so these fire ~10s earlier than the
# lagging set and, unlike them, fire DURING the burst rather than after it.
#
# v2.1 retuning, from 8,319 samples over 07-29/07-30:
#   php_rss 2000 -> 1500. The two real bursts we caught peaked at 2320 MB and
#   1590 MB, so 2000 missed one of them outright.
#   tcp_estab RETIRED as a trigger. It fired 3 dumps that caught nothing: all 28
#   apache-spawn storms in that window carried near-zero PHP (apache=153 with
#   php_rss=350MB), because bots hitting nonexistent .php get 404'd by Apache
#   without ever starting an interpreter. Apache child count is not the
#   mechanism. The column is still recorded, it just no longer trips a dump.
#   mysql_conn ADDED. It went 1 -> 19 at the exact peak of the 21:51 burst and
#   held there for a minute afterwards, so it is a real signal.
T_PHP_PROCS=25          # concurrent php workers
T_PHP_RSS=1500          # total php RSS in MB
T_MYSQL_CONN=15         # Threads_connected (sampled every 6th tick)

# domlog scan window. DOMLOG_WIN_MIN drives BOTH the find -mmin filter and the
# per-request timestamp match, so they cannot drift apart.
DOMLOG_WIN_MIN=3
DOMLOG_TAIL=5000        # lines read per log; must comfortably exceed WIN_MIN of traffic
STATIC_RE='\.(css|js|png|jpe?g|gif|svg|ico|woff2?|ttf|eot|webp|avif|map)'

while [ $# -gt 0 ]; do
	case "$1" in
		-i|--interval) INTERVAL="$2"; shift 2 ;;
		-d|--dir)      OUTDIR="$2"; shift 2 ;;
		-r|--retain)   RETAIN_DAYS="$2"; shift 2 ;;
		--status)      ACTION="status"; shift ;;
		--gaps)        ACTION="gaps"; shift ;;
		--stop)        ACTION="stop"; shift ;;
		-h|--help)     sed -n '2,87p' "$0" | sed 's/^# \{0,1\}//'; exit 1 ;;
		*) echo "Unknown option: $1" >&2; exit 1 ;;
	esac
done

mkdir -p "$OUTDIR" 2>/dev/null || { echo "Cannot create $OUTDIR" >&2; exit 1; }
PIDFILE="$OUTDIR/watch.pid"
NCPU=$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1)

samplefile() { echo "$OUTDIR/samples-$(date +%Y-%m-%d).tsv"; }

# cPanel has moved domlogs around across versions, and EA4 splits them per user.
# Collect every layout we might be on; callers filter by mtime.
# Override with DOMLOG_DIRS="/path/a /path/b" for a non-standard box or to test.
domlog_dirs() {
	for d in ${DOMLOG_DIRS:-/etc/apache2/logs/domlogs \
	                        /var/log/apache2/domlogs \
	                        /usr/local/apache/domlogs}; do
		[ -d "$d" ] && echo "$d"
	done
}

# Every access log currently taking traffic, ONE ENTRY PER PHYSICAL FILE.
#
# v2 ranked the same vhost 2-4 times. cPanel keeps both a top-level
# domlogs/<domain> entry and a per-user domlogs/<user>/<domain> entry for the
# same file, and the candidate dirs above can themselves resolve to one tree, so
# a plain find returned the same log under several paths. That inflated the
# per-vhost counts AND skewed the top-IP table unevenly (a log counted 4x
# contributed its IPs 4x), which is what made the whole table unusable.
#
# Dedupe on inode (%i), which is correct for both the hardlink and the
# symlinked-directory layout. Sort depth (%d) descending so the survivor is the
# per-user path — that one carries the cPanel account name, which is the single
# most valuable field in the whole dump.
#
# The exclusion list also had a hole: '*-bytes_log' does not match
# 'kurri-imapbytes_log', so mail byte-counters were being ranked as vhosts.
domlog_files() {
	for d in $(domlog_dirs); do
		find "$d" -maxdepth 2 -type f -mmin -"$DOMLOG_WIN_MIN" -size +0 \
			! -name '*.gz' ! -name '*.bz2' ! -name '*.zip' \
			! -name '*bytes_log' ! -name '*.offset' \
			! -name '*.bkup' ! -name '*.bak' ! -name '*.localhost' \
			! -name 'proxy-subdomains-vhost.*' \
			-printf '%i\t%d\t%p\n' 2>/dev/null
	done | sort -t"$(printf '\t')" -k1,1n -k2,2nr | awk -F'\t' '!seen[$1]++ { print $3 }'
}

# Apache timestamps look like [30/Jul/2026:05:32:02 +1000]. Emit one
# 'dd/Mon/yyyy:HH:MM' pattern per minute in the window so requests can be
# counted by TIME rather than by line position.
#
# v2 counted non-static lines inside `tail -200`. Any log with 200+ lines
# therefore reported exactly 200 — every row in the table read the same number.
# It measured log length, not current traffic, and ranked nothing.
win_pats() {
	m=0
	while [ "$m" -lt "$DOMLOG_WIN_MIN" ]; do
		date -d "-${m} min" +'%d/%b/%Y:%H:%M' 2>/dev/null
		m=$((m+1))
	done
}

# Requests in the window, from an already-tailed stream on stdin.
win_reqs() { grep -F -f "$1" 2>/dev/null | grep -vE "$STATIC_RE"; }

# This process and every ancestor. Any pgrep -f search can match a shell that is
# merely CARRYING the search pattern in its own argv (and a forked subshell has
# its own PID, so excluding $$ alone does not help). Filtering the whole
# ancestry makes such a match structurally impossible.
selfpids() {
	p=$$
	while [ -n "$p" ] && [ "$p" -gt 1 ] 2>/dev/null; do
		echo "$p"
		p=$(awk '/^PPid:/{print $2}' "/proc/$p/status" 2>/dev/null)
	done
}

###############################################################################
# --stop
###############################################################################
if [ "$ACTION" = "stop" ]; then
	if [ -r "$PIDFILE" ] && kill -0 "$(cat "$PIDFILE")" 2>/dev/null; then
		kill "$(cat "$PIDFILE")" && echo "Stopped watcher pid $(cat "$PIDFILE")"
		rm -f "$PIDFILE"
	else
		echo "No running watcher found."
	fi
	exit 0
fi

###############################################################################
# --gaps : find outages (missing samples) and show the run-up to each
###############################################################################
if [ "$ACTION" = "gaps" ]; then
	echo "OUTAGE GAPS — periods where the recorder stopped being scheduled"
	echo "================================================================"
	echo "(a gap >= 3x the sample interval means the OS stopped running us)"
	echo
	for f in "$OUTDIR"/samples-*.tsv; do
		[ -r "$f" ] || continue
		awk -F'\t' -v iv="$INTERVAL" -v file="$(basename "$f")" -v dir="$OUTDIR" '
		NR==1 && $1=="iso" { hdr=1; next }
		{
			if (prev_e && ($2 - prev_e) > iv*3) {
				printf "\n%s  GAP  %s  ->  %s   (%d seconds unresponsive)\n", file, prev_t, $1, $2-prev_e
				print  "  LAST SAMPLE BEFORE THE GAP:"
				printf "    load1=%s  blocked(D)=%s  mem_avail=%s%%  swap_used=%sMB  swapout/s=%s  iowait=%s%%  steal=%s%%\n", \
					prev[3], prev[6], prev[9], prev[11], prev[13], prev[14], prev[15]
				printf "    mysql_running=%s  mysql_conn=%s  mysqld_rss=%sMB  php_procs=%s  php_rss=%sMB  wkhtml=%s\n", \
					prev[16], prev[17], prev[18], prev[19], prev[20], prev[22]
				# v2 columns are appended, so older files simply have no value here
				if (prev[24] != "" && prev[24] != "-")
					printf "    heaviest php user: %s  (%s procs, %s MB)\n", prev[24], prev[25], prev[26]
				print  "  FIRST SAMPLE AFTER:"
				printf "    load1=%s  blocked(D)=%s  mem_avail=%s%%  swap_used=%sMB\n", $3, $6, $9, $11

				# --- which signals actually tripped, before any verdict ---
				mem_low   = (prev[9]+0  < 25)
				mem_crit  = (prev[9]+0  < 12)
				thrash    = (prev[13]+0 > 200)
				thrash_hi = (prev[13]+0 > 800)
				dstate    = (prev[6]+0  >= 10)
				iow       = (prev[14]+0 >= 30)
				steal     = (prev[15]+0 >= 20)
				phpburst  = (prev[19]+0 >= 25 || prev[20]+0 >= 2000)
				quiet     = (prev[9]+0 > 40 && prev[6]+0 < 5 && prev[13]+0 < 50 && prev[14]+0 < 10)

				sig = ""
				if (mem_low)  sig = sig sprintf("  mem_avail=%s%%", prev[9])
				if (thrash)   sig = sig sprintf("  swapout=%s/s", prev[13])
				if (dstate)   sig = sig sprintf("  blocked=%s", prev[6])
				if (iow)      sig = sig sprintf("  iowait=%s%%", prev[14])
				if (steal)    sig = sig sprintf("  steal=%s%%", prev[15])
				if (phpburst) sig = sig sprintf("  php=%s procs/%s MB", prev[19], prev[20])
				printf "  SIGNALS TRIPPED:%s\n", (sig == "" ? "  none" : sig)

				# --- verdict. Memory first: swap thrash pins every process in D
				#     state and drives iowait up, so those two must NOT be read
				#     as a storage fault while memory is also under pressure.
				if (phpburst && (mem_crit || mem_low || thrash_hi))
					v = "PHP WORKER BURST -> memory exhaustion. A vhost took a concurrency spike and forked enough php workers to eat the box. The per-user rollup and vhost traffic table in the incident dump name the account."
				else if (mem_crit || thrash_hi)
					v = "MEMORY EXHAUSTION -> swap death spiral. The whole OS stalls; expect an OOM kill at the end of the gap. Cross-check: grep -i oom-kill /var/log/messages"
				else if (mem_low && thrash)
					v = "MEMORY PRESSURE building -> heading for the same spiral. Whatever allocated is named in the incident dump below."
				else if (dstate && iow && !mem_low && !thrash)
					v = "IO / STORAGE STALL (D-state + iowait with memory genuinely fine) -> disk or host storage, not the ERP"
				else if (steal && !mem_low && !dstate)
					v = "HYPERVISOR CONTENTION (CPU stolen, memory and IO fine) -> VMware host / noisy neighbour, raise with the provider"
				else if (quiet)
					v = "NO LOCAL DEGRADATION in the last sample -> host-level event (VM pause, snapshot, live migration), NOT the ERP. Check the VMware side."
				else
					v = "INCONCLUSIVE from this sample — read the incident dump, and note the sample may simply predate the real onset (shorten -i)"
				printf "  => LIKELY: %s\n", v

				# point at the incident dump that covers this gap, if any
				ni = 0
				cmd = "ls " dir "/incident-*.txt 2>/dev/null"
				while ((cmd | getline f) > 0) inc[++ni] = f
				close(cmd)
				if (ni > 0) {
					print "  INCIDENT DUMPS on disk (these name the processes that ate the RAM):"
					for (k=1; k<=ni; k++) printf "    %s\n", inc[k]
				}
			}
			prev_e=$2; prev_t=$1
			for (i=1;i<=NF;i++) prev[i]=$i
		}' "$f"
	done
	echo
	exit 0
fi

###############################################################################
# --status : today's peaks
###############################################################################
if [ "$ACTION" = "status" ]; then
	f="$(samplefile)"
	if [ ! -r "$f" ]; then echo "No samples today at $f"; exit 1; fi
	if [ -r "$PIDFILE" ] && kill -0 "$(cat "$PIDFILE")" 2>/dev/null; then
		echo "Watcher: RUNNING (pid $(cat "$PIDFILE"), interval ${INTERVAL}s)"
	else
		echo "Watcher: NOT RUNNING"
	fi
	echo "File   : $f  ($(wc -l < "$f") samples)"
	echo
	awk -F'\t' 'BEGIN { ml=-1; lo=1e9; mb=-1; mr=-1; sw=-1; st=-1; pp=-1; pr=-1; tc=-1
		mlt="n/a"; lot="n/a"; mbt="n/a"; mrt="n/a"; swt="n/a"; stt="n/a"; ppt="n/a"; prt="n/a"; tct="n/a"; pru="-" }
	NR>1 {
		if ($3+0  > ml) { ml=$3+0;  mlt=$1 }
		if ($9+0  < lo) { lo=$9+0;  lot=$1 }
		if ($6+0  > mb) { mb=$6+0;  mbt=$1 }
		if ($16+0 > mr) { mr=$16+0; mrt=$1 }
		if ($11+0 > sw) { sw=$11+0; swt=$1 }
		if ($15+0 > st) { st=$15+0; stt=$1 }
		if ($19+0 > pp) { pp=$19+0; ppt=$1 }
		if ($20+0 > pr) { pr=$20+0; prt=$1; pru=($24==""?"-":$24) }
		if ($23+0 > tc) { tc=$23+0; tct=$1 }
	} END {
		if (ml<0) { print "  (no samples yet)"; exit }
		printf "  peak load1        : %-8s at %s\n", ml, mlt
		printf "  min mem available : %-7s%% at %s\n", lo, lot
		printf "  peak blocked (D)  : %-8s at %s\n", mb, mbt
		printf "  peak swap used    : %-6s MB at %s\n", sw, swt
		printf "  peak mysql running: %-8s at %s\n", mr, mrt
		printf "  peak cpu steal    : %-7s%% at %s\n", st, stt
		printf "  peak php procs    : %-8s at %s\n", pp, ppt
		printf "  peak php rss      : %-6s MB at %s   (heaviest user: %s)\n", pr, prt, pru
		printf "  peak tcp estab    : %-8s at %s\n", tc, tct
	}' "$f"
	echo
	echo "Incidents captured today:"
	ls -la "$OUTDIR"/incident-"$(date +%Y-%m-%d)"* 2>/dev/null || echo "  none"
	echo
	echo "Run --gaps to detect outage windows across all days."
	exit 0
fi

###############################################################################
# RUN
###############################################################################

# v2.2 — a kernel lock, not just a pidfile.
# The 08-12..08-27 dataset was recorded by TWO concurrent watchers: 17,190 rows
# per day against the 8,640 expected at a 10s interval, every timestamp present
# exactly twice with slightly different values. That doubles the IO this tool
# imposes on an already-struggling box, and silently double-counts for anything
# that reads the TSV without deduping.
#
# The pidfile check below cannot prevent it on its own: the TERM trap deletes
# the file, so a restart race -- or an @reboot cron entry plus a manual start --
# finds no pidfile and happily starts a second copy. flock is held by the
# kernel for the life of the process and cannot go stale.
# Keep the lock OUTSIDE the data directory. A lock file living in $OUTDIR is
# defeated the moment anything removes or recreates that directory: flock holds
# the INODE, so a deleted lock file leaves the old instance holding an orphan
# while a new instance happily locks the freshly created one. (Reproduced in
# testing -- an `rm -rf` of the output dir let a second watcher start.)
for _lk in /var/lock /run/lock /var/run /tmp; do
	[ -d "$_lk" ] && [ -w "$_lk" ] && { LOCKFILE="$_lk/forensic-watch.lock"; break; }
done
LOCKFILE="${LOCKFILE:-$OUTDIR/watch.lock}"
if command -v flock >/dev/null 2>&1; then
	exec 9>"$LOCKFILE" 2>/dev/null || true
	if ! flock -n 9; then
		echo "Another watcher already holds $LOCKFILE — refusing to start a second instance." >&2
		echo "Running watchers: $(pgrep -fc 'forensic-watch.sh' 2>/dev/null)" >&2
		exit 1
	fi
else
	# No flock (unlikely, but do not fail open -- failing open is how we ended
	# up with two instances and a fortnight of doubled samples).
	_others=$(pgrep -f 'forensic-watch.sh' 2>/dev/null | grep -vc "^$$\$")
	if [ "${_others:-0}" -gt 0 ]; then
		echo "flock unavailable and $_others other forensic-watch.sh process(es) are running — refusing." >&2
		exit 1
	fi
fi

if [ -r "$PIDFILE" ] && kill -0 "$(cat "$PIDFILE")" 2>/dev/null; then
	echo "Already running as pid $(cat "$PIDFILE"). Use --stop first." >&2
	exit 1
fi
echo $$ > "$PIDFILE"
trap 'rm -f "$PIDFILE"; exit 0' INT TERM

HEADER="iso	epoch	load1	load5	running	blocked	mem_total_mb	mem_avail_mb	mem_avail_pct	swap_total_mb	swap_used_mb	swapin_s	swapout_s	iowait_pct	steal_pct	mysql_running	mysql_conn	mysqld_rss_mb	php_procs	php_rss_mb	apache_procs	wkhtml_procs	tcp_estab	php_top_user	php_top_procs	php_top_rss_mb"

prev_cpu_total=0; prev_cpu_iowait=0; prev_cpu_steal=0
prev_swpin=0; prev_swpout=0; prev_ts=0
tick=0
mysql_run=""; mysql_conn=""

# probe mysql auth once
MYSQL_OK=0
if command -v mysql >/dev/null 2>&1; then
	mysql --connect-timeout=2 -N -B -e "SELECT 1" >/dev/null 2>&1 && MYSQL_OK=1
fi

while :; do
	F="$(samplefile)"
	# v2 appended three columns. If today's file was started by v1, roll it aside
	# rather than silently mixing row widths in one file.
	if [ -f "$F" ]; then
		if [ "$(head -1 "$F")" != "$HEADER" ]; then
			mv "$F" "${F%.tsv}-v1.tsv" 2>/dev/null
			printf '%s\n' "$HEADER" > "$F"
		fi
	else
		printf '%s\n' "$HEADER" > "$F"
	fi

	NOW_E=$(date +%s)
	NOW_T=$(date '+%Y-%m-%d %H:%M:%S')

	# --- load / procs -------------------------------------------------------
	read -r L1 L5 _ PR _ < /proc/loadavg
	RUNNING=$(awk '/^procs_running/{print $2}' /proc/stat)
	BLOCKED=$(awk '/^procs_blocked/{print $2}' /proc/stat)

	# --- memory -------------------------------------------------------------
	eval "$(awk '
		/^MemTotal:/     {mt=$2}
		/^MemAvailable:/ {ma=$2}
		/^SwapTotal:/    {st=$2}
		/^SwapFree:/     {sf=$2}
		END {
			printf "MEM_T=%d; MEM_A=%d; SWP_T=%d; SWP_U=%d; MEM_P=%.1f\n",
				mt/1024, ma/1024, st/1024, (st-sf)/1024, (mt>0? ma*100/mt : 0)
		}' /proc/meminfo)"

	# --- cpu deltas (iowait / steal) ---------------------------------------
	read -r _ u n s i io irq sirq steal _ < /proc/stat
	cpu_total=$((u+n+s+i+io+irq+sirq+steal))
	if [ "$prev_cpu_total" -gt 0 ]; then
		dt=$((cpu_total - prev_cpu_total))
		if [ "$dt" -gt 0 ]; then
			IOWAIT=$(awk -v a="$((io - prev_cpu_iowait))" -v b="$dt" 'BEGIN{printf "%.1f", a*100/b}')
			STEAL=$(awk -v a="$((steal - prev_cpu_steal))" -v b="$dt" 'BEGIN{printf "%.1f", a*100/b}')
		else IOWAIT=0; STEAL=0; fi
	else IOWAIT=0; STEAL=0; fi
	prev_cpu_total=$cpu_total; prev_cpu_iowait=$io; prev_cpu_steal=$steal

	# --- swap rate ----------------------------------------------------------
	swpin=$(awk '/^pswpin/{print $2}' /proc/vmstat)
	swpout=$(awk '/^pswpout/{print $2}' /proc/vmstat)
	if [ "$prev_ts" -gt 0 ]; then
		el=$((NOW_E - prev_ts)); [ "$el" -lt 1 ] && el=1
		SWIN=$(( (swpin - prev_swpin) / el ))
		SWOUT=$(( (swpout - prev_swpout) / el ))
	else SWIN=0; SWOUT=0; fi
	prev_swpin=$swpin; prev_swpout=$swpout; prev_ts=$NOW_E

	# --- process classes ----------------------------------------------------
	# ONE ps call for every class, plus per-user php attribution. user:20 stops
	# cPanel account names being truncated to "evoluti+" the way ps defaults to.
	eval "$(ps -eo rss=,user:20=,comm= 2>/dev/null | awk '
		$3 ~ /^(mysqld|mariadbd)$/          { my  += $1 }
		$3 ~ /^(php-cgi|php-fpm|lsphp|php)$/{ pn++; prss += $1; un[$2]++; ur[$2] += $1 }
		$3 ~ /^(httpd|apache2)$/            { an++ }
		$3 ~ /wkhtmlto/                     { wk++ }
		END {
			tu = "-"; tr = 0; tn = 0
			for (u in ur) if (ur[u] > tr) { tr = ur[u]; tu = u; tn = un[u] }
			gsub(/[^A-Za-z0-9._-]/, "", tu)
			printf "MYSQLD_RSS=%d; PHP_N=%d; PHP_RSS=%d; APACHE_N=%d; WKHTML_N=%d; PHP_TOP_U=\"%s\"; PHP_TOP_N=%d; PHP_TOP_RSS=%d\n",
				my/1024, pn+0, prss/1024, an+0, wk+0, tu, tn+0, tr/1024
		}')"
	TCP_EST=$( (ss -tan 2>/dev/null || netstat -tan 2>/dev/null) | grep -c ESTAB )

	# --- threshold check ----------------------------------------------------
	TRIP=0; WHY=""
	# lagging
	awk -v v="$MEM_P" -v t="$T_MEM_AVAIL_PCT" 'BEGIN{exit !(v<t)}' && { TRIP=1; WHY="$WHY mem_avail=${MEM_P}%"; }
	awk -v v="$L1" -v c="$NCPU" -v t="$T_LOAD_PER_CPU" 'BEGIN{exit !(v/c>t)}' && { TRIP=1; WHY="$WHY load=${L1}/${NCPU}cpu"; }
	[ "${BLOCKED:-0}" -ge "$T_BLOCKED" ] && { TRIP=1; WHY="$WHY blocked=$BLOCKED"; }
	[ "${SWOUT:-0}"   -ge "$T_SWAPOUT" ] && { TRIP=1; WHY="$WHY swapout=$SWOUT/s"; }
	# leading — these are what actually catch a 20-second worker burst.
	# tcp_estab is deliberately NOT a trigger; see the threshold block above.
	[ "${PHP_N:-0}"    -ge "$T_PHP_PROCS" ] && { TRIP=1; WHY="$WHY php_procs=$PHP_N"; }
	[ "${PHP_RSS:-0}"  -ge "$T_PHP_RSS" ]   && { TRIP=1; WHY="$WHY php_rss=${PHP_RSS}MB"; }

	# --- mysql (every 6th tick, or immediately when tripped) ----------------
	tick=$((tick+1))
	if [ "$MYSQL_OK" = "1" ] && { [ "$TRIP" = "1" ] || [ $((tick % 6)) -eq 1 ]; }; then
		mysql_run=$(mysql --connect-timeout=2 -N -B -e \
			"SELECT VARIABLE_VALUE FROM information_schema.GLOBAL_STATUS WHERE VARIABLE_NAME='Threads_running'" 2>/dev/null)
		mysql_conn=$(mysql --connect-timeout=2 -N -B -e \
			"SELECT VARIABLE_VALUE FROM information_schema.GLOBAL_STATUS WHERE VARIABLE_NAME='Threads_connected'" 2>/dev/null)
	fi

	# Connection high-water is a leading signal too, but it is only available on
	# the ticks we actually queried, so it is checked after the mysql block. When
	# it trips on a non-dump tick the next tick re-queries and the dump follows.
	case "${mysql_conn:-}" in
		''|*[!0-9]*) : ;;
		*) [ "$mysql_conn" -ge "$T_MYSQL_CONN" ] && { TRIP=1; WHY="$WHY mysql_conn=$mysql_conn"; } ;;
	esac

	printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
		"$NOW_T" "$NOW_E" "$L1" "$L5" "${RUNNING:-0}" "${BLOCKED:-0}" \
		"$MEM_T" "$MEM_A" "$MEM_P" "$SWP_T" "$SWP_U" "$SWIN" "$SWOUT" \
		"$IOWAIT" "$STEAL" "${mysql_run:--}" "${mysql_conn:--}" "${MYSQLD_RSS:-0}" \
		"${PHP_N:-0}" "${PHP_RSS:-0}" "${APACHE_N:-0}" "${WKHTML_N:-0}" "${TCP_EST:-0}" \
		"${PHP_TOP_U:--}" "${PHP_TOP_N:-0}" "${PHP_TOP_RSS:-0}" >> "$F"

	# --- incident dump ------------------------------------------------------
	if [ "$TRIP" = "1" ]; then
		LAST="$OUTDIR/.last_incident"
		LASTE=0; [ -r "$LAST" ] && LASTE=$(cat "$LAST")
		if [ $((NOW_E - LASTE)) -ge 60 ]; then      # at most one dump per minute
			echo "$NOW_E" > "$LAST"
			I="$OUTDIR/incident-$(date +%Y-%m-%d_%H%M%S).txt"
			{
				echo "INCIDENT $NOW_T   trigger:$WHY"
				echo "=================================================================="
				echo "load: $(cat /proc/loadavg)   ncpu:$NCPU"
				echo "mem_avail: ${MEM_A}MB/${MEM_T}MB (${MEM_P}%)  swap_used:${SWP_U}MB  swapout:${SWOUT}/s"
				echo "blocked(D):$BLOCKED running:$RUNNING iowait:${IOWAIT}% steal:${STEAL}%"
				echo "php: ${PHP_N} procs / ${PHP_RSS}MB   heaviest user: ${PHP_TOP_U} (${PHP_TOP_N} procs, ${PHP_TOP_RSS}MB)"
				echo "apache: ${APACHE_N} procs   tcp_estab: ${TCP_EST}"
				echo

				# The single most useful table in this file: who owns the RAM.
				# Covers EVERY process, so nothing is lost to a top-N cutoff.
				echo "--- RSS by user (all processes) ---"
				printf '  %-20s %8s %12s\n' "USER" "PROCS" "RSS_MB"
				ps -eo rss=,user:20= 2>/dev/null | awk '
					{ n[$2]++; r[$2] += $1 }
					END { for (u in r) printf "  %-20s %8d %12d\n", u, n[u], r[u]/1024 }' \
					| sort -k3 -rn | head -20
				echo

				echo "--- php workers by user ---"
				printf '  %-20s %8s %12s\n' "USER" "PROCS" "RSS_MB"
				ps -eo rss=,user:20=,comm= 2>/dev/null | awk '
					$3 ~ /^(php-cgi|php-fpm|lsphp|php)$/ { n[$2]++; r[$2] += $1 }
					END { for (u in r) printf "  %-20s %8d %12d\n", u, n[u], r[u]/1024 }' \
					| sort -k3 -rn | head -20
				echo

				# The 2026-07-29 22:18 event lost 2.9 GB of MemAvailable and got
				# it all back inside 30s with only 427 MB of PHP on the box, at
				# 18% iowait and 2366 pages/s swapped out. Anon memory cannot
				# explain that; page-cache eviction by a big sequential reader
				# can. Nothing in v2 recorded enough to tell the two apart.
				echo "--- vm tuning / memory pressure ---"
				for k in swappiness vfs_cache_pressure min_free_kbytes overcommit_memory overcommit_ratio; do
					[ -r "/proc/sys/vm/$k" ] && printf '  vm.%-20s %s\n' "$k" "$(cat "/proc/sys/vm/$k" 2>/dev/null)"
				done
				awk '/^(Cached|Dirty|Writeback|Committed_AS|CommitLimit|SReclaimable|Active\(file\)|Inactive\(file\)):/ \
					{ printf "  %-24s %d MB\n", $1, $2/1024 }' /proc/meminfo 2>/dev/null
				for p in cpu memory io; do
					[ -r "/proc/pressure/$p" ] && printf '  PSI %-6s %s\n' "$p" "$(tr '\n' ' ' < "/proc/pressure/$p" 2>/dev/null)"
				done
				echo

				# Page-cache evictors. cPanel's nightly backup, a mysqldump, or a
				# Wordfence/ClamAV file walk will all do this, and 22:18 is inside
				# the usual backup window.
				# Every alternative has its first character bracketed. A regex like
				# [c]pbackup matches the string "cpbackup" but NOT the literal
				# text "[c]pbackup", so this pattern can never match the shell or
				# pipeline that is carrying it. Without that, any process whose
				# argv contains the pattern (this script under `bash -c`, or the
				# subshell of the command substitution — which has its own PID, so
				# excluding $$ does not help) reports itself as a finding.
				echo "--- backup / dump / scan processes (page-cache evictors) ---"
				SELFP=$(selfpids)
				EVP=$(pgrep -f '[c]pbackup|[p]kgacct|[b]ackup|[m]ysqldump|[m]ariabackup|[x]trabackup|[r]sync|[r]diff|[d]uplicity|[r]estic|[b]org|[c]lamscan|[c]lamd|[w]ordfence|[i]munify|[u]pdatedb|[m]locate|(^|/)[t]ar( |$)|[g]zip|[b]zip2|[x]z ' 2>/dev/null \
					| grep -vxF "${SELFP:-__none__}" | tr '\n' ',' | sed 's/,$//')
				EVOUT=""
				# PIDs can die between pgrep and ps, so re-check rather than
				# printing a bare header with no rows under it.
				[ -n "$EVP" ] && EVOUT=$(ps -o pid,user:20,etime,rss,args -p "$EVP" 2>/dev/null | tail -n +2 | head -20 | cut -c1-200)
				if [ -n "$EVOUT" ]; then
					printf '  %-8s %-20s %-9s %8s %s\n' "PID" "USER" "ELAPSED" "RSS_KB" "COMMAND"
					echo "$EVOUT" | sed 's/^/  /'
				else
					echo "  (none — no backup/dump/scan process running at trip time)"
				fi
				echo

				echo "--- top 60 by RSS ---"
				ps -eo pid,ppid,user:20,rss,pcpu,stat,etime,args --sort=-rss 2>/dev/null | head -61 | cut -c1-200
				echo
				# ---------------------------------------------------------------
				# v2.2 — NAME THE URL.
				#
				# Every previous round ended at "we know WHICH ACCOUNT, not WHICH
				# REQUEST". The reason is structural, not an oversight: Apache
				# writes the access log only when a request COMPLETES, so an
				# in-flight burst is invisible to any domlog-based ranking. On
				# 08-03 madalice held 4 GB of php-cgi with ZERO logged requests.
				#
				# Under the EA4 cgi handler each request gets its own interpreter,
				# and Apache passes the CGI environment to it. So the answer is
				# sitting in /proc/<pid>/environ of the live workers the whole
				# time. Read it while they are still running and the burst names
				# itself -- URL, query string, client IP, vhost -- with no
				# dependency on the request ever finishing.
				# ---------------------------------------------------------------
				ATTR_USERS="${PHP_TOP_U:--}"
				case " $ATTR_USERS " in *" evolution "*) ;; *) ATTR_USERS="$ATTR_USERS evolution" ;; esac
				for au in $ATTR_USERS; do
					[ "$au" = "-" ] && continue
					APIDS=$(ps -eo pid,user:20,rss,etime,comm --no-headers 2>/dev/null \
						| awk -v u="$au" '$2==u && $5 ~ /^(php-cgi|php-fpm|lsphp|php)$/ {print $1}' | head -60)
					[ -z "$APIDS" ] && continue

					echo "--- LIVE REQUEST ATTRIBUTION: $au (/proc/<pid>/environ) ---"
					ATT="$OUTDIR/.attr.$$"
					: > "$ATT"
					for p in $APIDS; do
						[ -r "/proc/$p/environ" ] || continue
						tr '\0' '\n' < "/proc/$p/environ" 2>/dev/null | awk -v p="$p" '
							/^HTTP_HOST=/       { sub(/^HTTP_HOST=/,"");       h=$0 }
							/^REQUEST_METHOD=/  { sub(/^REQUEST_METHOD=/,"");  m=$0 }
							/^REQUEST_URI=/     { sub(/^REQUEST_URI=/,"");     u=$0 }
							/^QUERY_STRING=/    { sub(/^QUERY_STRING=/,"");    q=$0 }
							/^SCRIPT_FILENAME=/ { sub(/^SCRIPT_FILENAME=/,""); s=$0 }
							/^REMOTE_ADDR=/     { sub(/^REMOTE_ADDR=/,"");     r=$0 }
							END { if (u != "" || s != "")
								printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\n", p, (h?h:"-"), (m?m:"-"), (u?u:"-"), (q?q:"-"), (s?s:"-"), (r?r:"-") }' >> "$ATT" 2>/dev/null
					done

					if [ ! -s "$ATT" ]; then
						echo "  (no readable environ — workers may be php-fpm, which does not"
						echo "   carry the CGI env per child; use the fpm slow/status log instead)"
						echo
						rm -f "$ATT"
						continue
					fi

					# Aggregate FIRST. 40 hits on one endpoint and 40 distinct
					# one-off scanner paths look identical in a raw list and are
					# opposite diagnoses.
					# Keep the FIRST query parameter in the grouping key. The ERP
					# front-controllers everything through /index.php?page=<x>, so
					# stripping the query entirely would collapse every report on
					# the box into one meaningless row. Same for WordPress
					# ?rest_route= and ?s= floods.
					echo "  -- concurrent requests grouped by script + path (the burst, aggregated) --"
					awk -F'\t' '{ u=$4; q=""; i=index(u,"?");
						if (i>0) { q=substr(u,i+1); u=substr(u,1,i-1) }
						if (q != "") { split(q,pp,"&"); q="?" pp[1] }
						print $2" "$3" "u q }' "$ATT" \
						| sort | uniq -c | sort -rn | head -20 \
						| awk '{ c=$1; $1=""; sub(/^ /,""); printf "  %6s  %s\n", c, $0 }'
					echo
					echo "  -- client IPs driving them --"
					awk -F'\t' '{print $7}' "$ATT" | sort | uniq -c | sort -rn | head -10 \
						| awk '{printf "  %6s  %s\n", $1, $2}'
					echo
					echo "  -- longest-running workers (full URI + query + peer) --"
					for p in $(awk -F'\t' '{print $1}' "$ATT"); do
						et=$(ps -o etimes= -p "$p" 2>/dev/null | tr -d ' ')
						printf '%s\t%s\n' "${et:-0}" "$p"
					done | sort -k1,1rn | head -12 | while IFS="$(printf '\t')" read -r et p; do
						rs=$(ps -o rss= -p "$p" 2>/dev/null | tr -d ' ')
						awk -F'\t' -v p="$p" -v et="$et" -v rs="${rs:-0}" '$1==p {
							printf "  pid=%-8s %4ss  %6.1fMB  %s  %s %s%s\n", $1, et, rs/1024, $7, $3, $2, $4
							printf "      script=%s\n", $6
							if ($5 != "-" && $5 != "") printf "      query =%s\n", substr($5,1,300)
						}' "$ATT"
					done
					rm -f "$ATT"
					echo
				done

				echo "--- processes in D state (uninterruptible IO) ---"
				ps -eo pid,stat,wchan:24,args 2>/dev/null | awk '$2 ~ /D/' | head -30 | cut -c1-200
				echo
				if [ "$MYSQL_OK" = "1" ]; then
					echo "--- mariadb processlist (non-sleeping) ---"
					mysql --connect-timeout=2 -t -e \
						"SELECT ID,USER,DB,COMMAND,TIME,STATE,LEFT(INFO,300) q
						 FROM information_schema.PROCESSLIST
						 WHERE COMMAND<>'Sleep' ORDER BY TIME DESC LIMIT 40" 2>&1
					echo
					echo "--- lock waits ---"
					mysql --connect-timeout=2 -t -e \
						"SELECT ID,DB,TIME,STATE,LEFT(INFO,200) q FROM information_schema.PROCESSLIST
						 WHERE STATE LIKE '%lock%' OR STATE LIKE '%Waiting for table%' ORDER BY TIME DESC LIMIT 25" 2>&1
				fi
				echo

				# v1 only ever looked at the ERP domlog, which is why the account
				# that caused the 09:38 burst never appeared in any dump. This is
				# a ~20-account shared box; rank every vhost that is currently
				# taking traffic, then show the busiest ones in detail.
				echo "--- vhost traffic RIGHT NOW (last ${DOMLOG_WIN_MIN} min, deduped by inode) ---"
				LOGS=$(domlog_files)
				WINF="$OUTDIR/.winpats.$$"
				win_pats > "$WINF" 2>/dev/null
				if [ -z "$LOGS" ]; then
					echo "  (no recently-written domlogs found — check domlog_dirs())"
				else
					# Count once, reuse for every table below. Ranking by a real
					# time window means these numbers are comparable across
					# vhosts and cannot saturate the way tail -N did.
					RANK="$OUTDIR/.vhostrank.$$"
					: > "$RANK"
					for lg in $LOGS; do
						c=$(tail -n "$DOMLOG_TAIL" "$lg" 2>/dev/null | win_reqs "$WINF" | wc -l)
						[ "${c:-0}" -gt 0 ] && printf '%s\t%s\n' "${c:-0}" "$lg" >> "$RANK"
					done
					printf '  %-52s %10s\n' "VHOST LOG" "REQ/${DOMLOG_WIN_MIN}min"
					sort -k1,1rn "$RANK" | head -15 | while IFS="$(printf '\t')" read -r c lg; do
						printf '  %-52s %10s\n' "${lg##*/domlogs/}" "$c"
					done
					echo "  (total vhosts taking traffic: $(wc -l < "$RANK"))"
					echo

					echo "--- top client IPs across all vhosts (last ${DOMLOG_WIN_MIN} min) ---"
					for lg in $LOGS; do tail -n "$DOMLOG_TAIL" "$lg" 2>/dev/null; done \
						| win_reqs "$WINF" \
						| awk '{print $1}' | sort | uniq -c | sort -rn | head -15 \
						| awk '{printf "  %8s  %s\n", $1, $2}'
					echo

					# THE table this whole exercise exists for: name the URL.
					# "last N raw lines" never aggregated, so a scanner firing 40
					# distinct one-off paths looked identical to 40 hits on one
					# expensive endpoint. These two are the opposite diagnosis.
					echo "--- top request paths across all vhosts (last ${DOMLOG_WIN_MIN} min) ---"
					for lg in $LOGS; do tail -n "$DOMLOG_TAIL" "$lg" 2>/dev/null; done \
						| win_reqs "$WINF" \
						| awk -F'"' '{print $2}' \
						| awk '{ u=$2; sub(/\?.*/,"",u); if (u != "") print $1, u }' \
						| sort | uniq -c | sort -rn | head -20 \
						| awk '{printf "  %8s  %s %s\n", $1, $2, $3}'
					echo

					# v2.2 — the burst user's vhost is ALWAYS expanded, even when
					# it ranks below the top 3. The ranking counts COMPLETED
					# requests, so the account melting the box can sit at rank 9
					# with a handful of logged hits while holding 100 workers
					# in-flight. That is exactly how the top consumer stayed
					# unnamed for three rounds.
					EXP="$OUTDIR/.expand.$$"
					sort -k1,1rn "$RANK" | head -3 | cut -f2 > "$EXP"
					if [ -n "${PHP_TOP_U:-}" ] && [ "$PHP_TOP_U" != "-" ] \
					   && [ -r "/var/cpanel/users/$PHP_TOP_U" ]; then
						for d in $(awk -F= '/^DNS[0-9]*=/{print $2}' \
								"/var/cpanel/users/$PHP_TOP_U" 2>/dev/null); do
							[ -z "$d" ] && continue
							grep -F "/$d" "$RANK" 2>/dev/null | cut -f2 >> "$EXP"
						done
					fi
					sort -u "$EXP" -o "$EXP" 2>/dev/null
					echo "--- expanded vhosts (busiest 3 + burst user ${PHP_TOP_U:--}): top paths + last 10 requests ---"
					while read -r lg; do
						echo "  == ${lg##*/domlogs/} =="
						echo "     -- top paths --"
						tail -n "$DOMLOG_TAIL" "$lg" 2>/dev/null | win_reqs "$WINF" \
							| awk -F'"' '{print $2}' \
							| awk '{ u=$2; sub(/\?.*/,"",u); if (u != "") print $1, u }' \
							| sort | uniq -c | sort -rn | head -12 \
							| awk '{printf "     %8s  %s %s\n", $1, $2, $3}'
						echo "     -- last 10 requests --"
						tail -n "$DOMLOG_TAIL" "$lg" 2>/dev/null | win_reqs "$WINF" \
							| tail -10 | cut -c1-220 | sed 's/^/     /'
						echo
					done < "$EXP"
					rm -f "$EXP"
					rm -f "$RANK"
				fi
				rm -f "$WINF"

				# Inline the run-up so the dump is self-contained — you should not
				# have to go back to the TSV to see how the burst built.
				echo "--- last 30 samples (run-up to this incident) ---"
				awk -F'\t' 'NR>1{printf "  %s L1=%-6s run=%-4s blk=%-3s avail=%-5s(%s%%) swpO=%-6s conn=%-4s myRSS=%-5s php=%-4s phpRSS=%-6s ap=%-4s tcp=%-5s topuser=%s\n",$1,$3,$5,$6,$8,$9,$13,$17,$18,$19,$20,$21,$23,$24}' "$F" | tail -30
			} > "$I" 2>&1
			sync 2>/dev/null
		fi
	fi

	# --- retention ----------------------------------------------------------
	if [ $((tick % 360)) -eq 0 ]; then
		find "$OUTDIR" -maxdepth 1 -name 'samples-*.tsv' -mtime +"$RETAIN_DAYS" -delete 2>/dev/null
		find "$OUTDIR" -maxdepth 1 -name 'incident-*.txt' -mtime +"$RETAIN_DAYS" -delete 2>/dev/null
	fi

	sleep "$INTERVAL"
done