#!/bin/bash
###############################################################################
# forensic-watch.sh — always-on black-box recorder
#
# WHY THIS EXISTS
#   The box goes fully unresponsive (no SSH, no services) for ~5 min and then
#   recovers. You therefore CANNOT log in and capture in-flight state — by the
#   time you have a shell, the evidence is gone. This daemon samples cheap
#   /proc counters to disk continuously, so the next outage is recorded whether
#   or not anyone can log in.
#
#   Critically: if the OS hangs hard, this process stops being scheduled too.
#   That is itself the signal — the GAP in the sample file, plus the last
#   sample before it, tells us which failure mode fired:
#
#     last sample shows mem_avail collapsing + swap churn  -> memory exhaustion (our app)
#     last sample shows blocked(D-state) spiking, mem fine -> IO / storage stall
#     last sample shows steal% high, mem+io fine           -> hypervisor / noisy neighbour
#     no degradation at all, then a clean gap              -> host-level event, NOT us
#
#   That single distinction decides whether the OOM project is even the right
#   investigation. Nothing in the existing scripts could tell us.
#
# USAGE
#   nohup ./forensic-watch.sh > /dev/null 2>&1 &          # start (survives logout)
#   ./forensic-watch.sh -i 5 -d /var/log/erp-forensics    # 5s interval, custom dir
#   ./forensic-watch.sh --status                          # show today's peaks + gaps
#   ./forensic-watch.sh --gaps                            # list detected outages
#   ./forensic-watch.sh --stop
#
#   Survive reboot:  add to root crontab ->  @reboot /path/to/forensic-watch.sh -d /var/log/erp-forensics
#
# COST
#   ~1 syscall-cheap read of /proc per interval. MariaDB is only queried every
#   6th sample (or immediately when a threshold trips), with a 2s connect
#   timeout, so the recorder cannot itself contribute to the problem.
#
# Read-only. Never kills, restarts or reconfigures anything.
###############################################################################

INTERVAL=10
OUTDIR="$(pwd)/samples"
RETAIN_DAYS=14
ACTION="run"
PIDFILE=""

# incident thresholds
T_MEM_AVAIL_PCT=12      # MemAvailable below this % of MemTotal
T_LOAD_PER_CPU=8        # load1 / nproc above this
T_BLOCKED=10            # procs in uninterruptible (D) state
T_SWAPOUT=2000          # pages swapped out per second

while [ $# -gt 0 ]; do
	case "$1" in
		-i|--interval) INTERVAL="$2"; shift 2 ;;
		-d|--dir)      OUTDIR="$2"; shift 2 ;;
		-r|--retain)   RETAIN_DAYS="$2"; shift 2 ;;
		--status)      ACTION="status"; shift ;;
		--gaps)        ACTION="gaps"; shift ;;
		--stop)        ACTION="stop"; shift ;;
		-h|--help)     sed -n '2,42p' "$0" | sed 's/^# \{0,1\}//'; exit 1 ;;
		*) echo "Unknown option: $1" >&2; exit 1 ;;
	esac
done

mkdir -p "$OUTDIR" 2>/dev/null || { echo "Cannot create $OUTDIR" >&2; exit 1; }
PIDFILE="$OUTDIR/watch.pid"
NCPU=$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1)

samplefile() { echo "$OUTDIR/samples-$(date +%Y-%m-%d).tsv"; }

###############################################################################
# --stop
###############################################################################
if [ "$ACTION" = "stop" ]; then
	if [ -r "$PIDFILE" ] && kill -0 "$(cat "$PIDFILE")" 2>/dev/null; then
		kill "$(cat "$PIDFILE")" && echo "Stopped watcher pid $(cat "$PIDFILE")"
		rm -f "$PIDFILE"
	else
		echo "No running watcher found."
	fi
	exit 0
fi

###############################################################################
# --gaps : find outages (missing samples) and show the run-up to each
###############################################################################
if [ "$ACTION" = "gaps" ]; then
	echo "OUTAGE GAPS — periods where the recorder stopped being scheduled"
	echo "================================================================"
	echo "(a gap >= 3x the sample interval means the OS stopped running us)"
	echo
	for f in "$OUTDIR"/samples-*.tsv; do
		[ -r "$f" ] || continue
		awk -F'\t' -v iv="$INTERVAL" -v file="$(basename "$f")" -v dir="$OUTDIR" '
		NR==1 && $1=="iso" { hdr=1; next }
		{
			if (prev_e && ($2 - prev_e) > iv*3) {
				printf "\n%s  GAP  %s  ->  %s   (%d seconds unresponsive)\n", file, prev_t, $1, $2-prev_e
				print  "  LAST SAMPLE BEFORE THE GAP:"
				printf "    load1=%s  blocked(D)=%s  mem_avail=%s%%  swap_used=%sMB  swapout/s=%s  iowait=%s%%  steal=%s%%\n", \
					prev[3], prev[6], prev[9], prev[11], prev[13], prev[14], prev[15]
				printf "    mysql_running=%s  mysql_conn=%s  mysqld_rss=%sMB  php_procs=%s  php_rss=%sMB  wkhtml=%s\n", \
					prev[16], prev[17], prev[18], prev[19], prev[20], prev[22]
				print  "  FIRST SAMPLE AFTER:"
				printf "    load1=%s  blocked(D)=%s  mem_avail=%s%%  swap_used=%sMB\n", $3, $6, $9, $11

				# --- which signals actually tripped, before any verdict ---
				mem_low   = (prev[9]+0  < 25)
				mem_crit  = (prev[9]+0  < 12)
				thrash    = (prev[13]+0 > 200)
				thrash_hi = (prev[13]+0 > 800)
				dstate    = (prev[6]+0  >= 10)
				iow       = (prev[14]+0 >= 30)
				steal     = (prev[15]+0 >= 20)
				quiet     = (prev[9]+0 > 40 && prev[6]+0 < 5 && prev[13]+0 < 50 && prev[14]+0 < 10)

				sig = ""
				if (mem_low) sig = sig sprintf("  mem_avail=%s%%", prev[9])
				if (thrash)  sig = sig sprintf("  swapout=%s/s", prev[13])
				if (dstate)  sig = sig sprintf("  blocked=%s", prev[6])
				if (iow)     sig = sig sprintf("  iowait=%s%%", prev[14])
				if (steal)   sig = sig sprintf("  steal=%s%%", prev[15])
				printf "  SIGNALS TRIPPED:%s\n", (sig == "" ? "  none" : sig)

				# --- verdict. Memory first: swap thrash pins every process in D
				#     state and drives iowait up, so those two must NOT be read
				#     as a storage fault while memory is also under pressure.
				if (mem_crit || thrash_hi)
					v = "MEMORY EXHAUSTION -> swap death spiral. The whole OS stalls; expect an OOM kill at the end of the gap. Cross-check: grep -i oom-kill /var/log/messages"
				else if (mem_low && thrash)
					v = "MEMORY PRESSURE building -> heading for the same spiral. Whatever allocated is named in the incident dump below."
				else if (dstate && iow && !mem_low && !thrash)
					v = "IO / STORAGE STALL (D-state + iowait with memory genuinely fine) -> disk or host storage, not the ERP"
				else if (steal && !mem_low && !dstate)
					v = "HYPERVISOR CONTENTION (CPU stolen, memory and IO fine) -> VMware host / noisy neighbour, raise with the provider"
				else if (quiet)
					v = "NO LOCAL DEGRADATION in the last sample -> host-level event (VM pause, snapshot, live migration), NOT the ERP. Check the VMware side."
				else
					v = "INCONCLUSIVE from this sample — read the incident dump, and note the sample may simply predate the real onset (shorten -i)"
				printf "  => LIKELY: %s\n", v

				# point at the incident dump that covers this gap, if any
				ni = 0
				cmd = "ls " dir "/incident-*.txt 2>/dev/null"
				while ((cmd | getline f) > 0) inc[++ni] = f
				close(cmd)
				if (ni > 0) {
					print "  INCIDENT DUMPS on disk (these name the processes that ate the RAM):"
					for (k=1; k<=ni; k++) printf "    %s\n", inc[k]
				}
			}
			prev_e=$2; prev_t=$1
			for (i=1;i<=NF;i++) prev[i]=$i
		}' "$f"
	done
	echo
	exit 0
fi

###############################################################################
# --status : today's peaks
###############################################################################
if [ "$ACTION" = "status" ]; then
	f="$(samplefile)"
	if [ ! -r "$f" ]; then echo "No samples today at $f"; exit 1; fi
	if [ -r "$PIDFILE" ] && kill -0 "$(cat "$PIDFILE")" 2>/dev/null; then
		echo "Watcher: RUNNING (pid $(cat "$PIDFILE"), interval ${INTERVAL}s)"
	else
		echo "Watcher: NOT RUNNING"
	fi
	echo "File   : $f  ($(wc -l < "$f") samples)"
	echo
	awk -F'\t' 'BEGIN { ml=-1; lo=1e9; mb=-1; mr=-1; sw=-1; st=-1
		mlt="n/a"; lot="n/a"; mbt="n/a"; mrt="n/a"; swt="n/a"; stt="n/a" }
	NR>1 {
		if ($3+0 > ml) { ml=$3+0; mlt=$1 }
		if ($9+0 < lo) { lo=$9+0; lot=$1 }
		if ($6+0 > mb) { mb=$6+0; mbt=$1 }
		if ($16+0 > mr) { mr=$16+0; mrt=$1 }
		if ($11+0 > sw) { sw=$11+0; swt=$1 }
		if ($15+0 > st) { st=$15+0; stt=$1 }
	} END {
		if (ml<0) { print "  (no samples yet)"; exit }
		printf "  peak load1        : %-8s at %s\n", ml, mlt
		printf "  min mem available : %-7s%% at %s\n", lo, lot
		printf "  peak blocked (D)  : %-8s at %s\n", mb, mbt
		printf "  peak swap used    : %-6s MB at %s\n", sw, swt
		printf "  peak mysql running: %-8s at %s\n", mr, mrt
		printf "  peak cpu steal    : %-7s%% at %s\n", st, stt
	}' "$f"
	echo
	echo "Incidents captured today:"
	ls -la "$OUTDIR"/incident-"$(date +%Y-%m-%d)"* 2>/dev/null || echo "  none"
	echo
	echo "Run --gaps to detect outage windows across all days."
	exit 0
fi

###############################################################################
# RUN
###############################################################################

if [ -r "$PIDFILE" ] && kill -0 "$(cat "$PIDFILE")" 2>/dev/null; then
	echo "Already running as pid $(cat "$PIDFILE"). Use --stop first." >&2
	exit 1
fi
echo $$ > "$PIDFILE"
trap 'rm -f "$PIDFILE"; exit 0' INT TERM

HEADER="iso	epoch	load1	load5	running	blocked	mem_total_mb	mem_avail_mb	mem_avail_pct	swap_total_mb	swap_used_mb	swapin_s	swapout_s	iowait_pct	steal_pct	mysql_running	mysql_conn	mysqld_rss_mb	php_procs	php_rss_mb	apache_procs	wkhtml_procs	tcp_estab"

prev_cpu_total=0; prev_cpu_iowait=0; prev_cpu_steal=0
prev_swpin=0; prev_swpout=0; prev_ts=0
tick=0
mysql_run=""; mysql_conn=""

# probe mysql auth once
MYSQL_OK=0
if command -v mysql >/dev/null 2>&1; then
	mysql --connect-timeout=2 -N -B -e "SELECT 1" >/dev/null 2>&1 && MYSQL_OK=1
fi

while :; do
	F="$(samplefile)"
	[ -f "$F" ] || printf '%s\n' "$HEADER" > "$F"

	NOW_E=$(date +%s)
	NOW_T=$(date '+%Y-%m-%d %H:%M:%S')

	# --- load / procs -------------------------------------------------------
	read -r L1 L5 _ PR _ < /proc/loadavg
	RUNNING=$(awk '/^procs_running/{print $2}' /proc/stat)
	BLOCKED=$(awk '/^procs_blocked/{print $2}' /proc/stat)

	# --- memory -------------------------------------------------------------
	eval "$(awk '
		/^MemTotal:/     {mt=$2}
		/^MemAvailable:/ {ma=$2}
		/^SwapTotal:/    {st=$2}
		/^SwapFree:/     {sf=$2}
		END {
			printf "MEM_T=%d; MEM_A=%d; SWP_T=%d; SWP_U=%d; MEM_P=%.1f\n",
				mt/1024, ma/1024, st/1024, (st-sf)/1024, (mt>0? ma*100/mt : 0)
		}' /proc/meminfo)"

	# --- cpu deltas (iowait / steal) ---------------------------------------
	read -r _ u n s i io irq sirq steal _ < /proc/stat
	cpu_total=$((u+n+s+i+io+irq+sirq+steal))
	if [ "$prev_cpu_total" -gt 0 ]; then
		dt=$((cpu_total - prev_cpu_total))
		if [ "$dt" -gt 0 ]; then
			IOWAIT=$(awk -v a="$((io - prev_cpu_iowait))" -v b="$dt" 'BEGIN{printf "%.1f", a*100/b}')
			STEAL=$(awk -v a="$((steal - prev_cpu_steal))" -v b="$dt" 'BEGIN{printf "%.1f", a*100/b}')
		else IOWAIT=0; STEAL=0; fi
	else IOWAIT=0; STEAL=0; fi
	prev_cpu_total=$cpu_total; prev_cpu_iowait=$io; prev_cpu_steal=$steal

	# --- swap rate ----------------------------------------------------------
	swpin=$(awk '/^pswpin/{print $2}' /proc/vmstat)
	swpout=$(awk '/^pswpout/{print $2}' /proc/vmstat)
	if [ "$prev_ts" -gt 0 ]; then
		el=$((NOW_E - prev_ts)); [ "$el" -lt 1 ] && el=1
		SWIN=$(( (swpin - prev_swpin) / el ))
		SWOUT=$(( (swpout - prev_swpout) / el ))
	else SWIN=0; SWOUT=0; fi
	prev_swpin=$swpin; prev_swpout=$swpout; prev_ts=$NOW_E

	# --- process classes ----------------------------------------------------
	MYSQLD_RSS=$(ps -eo rss,comm 2>/dev/null | awk '$2 ~ /^(mysqld|mariadbd)$/ {s+=$1} END {printf "%d", s/1024}')
	PHP_N=$(ps -eo comm 2>/dev/null | grep -cE '^(php-cgi|php-fpm|lsphp|php)$')
	PHP_RSS=$(ps -eo rss,comm 2>/dev/null | awk '$2 ~ /^(php-cgi|php-fpm|lsphp|php)$/ {s+=$1} END {printf "%d", s/1024}')
	APACHE_N=$(ps -eo comm 2>/dev/null | grep -cE '^(httpd|apache2)$')
	WKHTML_N=$(ps -eo comm 2>/dev/null | grep -cE 'wkhtmlto')
	TCP_EST=$( (ss -tan 2>/dev/null || netstat -tan 2>/dev/null) | grep -c ESTAB )

	# --- threshold check ----------------------------------------------------
	TRIP=0; WHY=""
	awk -v v="$MEM_P" -v t="$T_MEM_AVAIL_PCT" 'BEGIN{exit !(v<t)}' && { TRIP=1; WHY="$WHY mem_avail=${MEM_P}%"; }
	awk -v v="$L1" -v c="$NCPU" -v t="$T_LOAD_PER_CPU" 'BEGIN{exit !(v/c>t)}' && { TRIP=1; WHY="$WHY load=${L1}/${NCPU}cpu"; }
	[ "${BLOCKED:-0}" -ge "$T_BLOCKED" ] && { TRIP=1; WHY="$WHY blocked=$BLOCKED"; }
	[ "${SWOUT:-0}" -ge "$T_SWAPOUT" ]   && { TRIP=1; WHY="$WHY swapout=$SWOUT/s"; }

	# --- mysql (every 6th tick, or immediately when tripped) ----------------
	tick=$((tick+1))
	if [ "$MYSQL_OK" = "1" ] && { [ "$TRIP" = "1" ] || [ $((tick % 6)) -eq 1 ]; }; then
		mysql_run=$(mysql --connect-timeout=2 -N -B -e \
			"SELECT VARIABLE_VALUE FROM information_schema.GLOBAL_STATUS WHERE VARIABLE_NAME='Threads_running'" 2>/dev/null)
		mysql_conn=$(mysql --connect-timeout=2 -N -B -e \
			"SELECT VARIABLE_VALUE FROM information_schema.GLOBAL_STATUS WHERE VARIABLE_NAME='Threads_connected'" 2>/dev/null)
	fi

	printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
		"$NOW_T" "$NOW_E" "$L1" "$L5" "${RUNNING:-0}" "${BLOCKED:-0}" \
		"$MEM_T" "$MEM_A" "$MEM_P" "$SWP_T" "$SWP_U" "$SWIN" "$SWOUT" \
		"$IOWAIT" "$STEAL" "${mysql_run:--}" "${mysql_conn:--}" "${MYSQLD_RSS:-0}" \
		"${PHP_N:-0}" "${PHP_RSS:-0}" "${APACHE_N:-0}" "${WKHTML_N:-0}" "${TCP_EST:-0}" >> "$F"

	# --- incident dump ------------------------------------------------------
	if [ "$TRIP" = "1" ]; then
		LAST="$OUTDIR/.last_incident"
		LASTE=0; [ -r "$LAST" ] && LASTE=$(cat "$LAST")
		if [ $((NOW_E - LASTE)) -ge 60 ]; then      # at most one dump per minute
			echo "$NOW_E" > "$LAST"
			I="$OUTDIR/incident-$(date +%Y-%m-%d_%H%M%S).txt"
			{
				echo "INCIDENT $NOW_T   trigger:$WHY"
				echo "=================================================================="
				echo "load: $(cat /proc/loadavg)   ncpu:$NCPU"
				echo "mem_avail: ${MEM_A}MB/${MEM_T}MB (${MEM_P}%)  swap_used:${SWP_U}MB  swapout:${SWOUT}/s"
				echo "blocked(D):$BLOCKED running:$RUNNING iowait:${IOWAIT}% steal:${STEAL}%"
				echo
				echo "--- top 30 by RSS ---"
				ps -eo pid,ppid,user,rss,pcpu,stat,etime,args --sort=-rss 2>/dev/null | head -31 | cut -c1-180
				echo
				echo "--- processes in D state (uninterruptible IO) ---"
				ps -eo pid,stat,wchan:24,args 2>/dev/null | awk '$2 ~ /D/' | head -30 | cut -c1-180
				echo
				if [ "$MYSQL_OK" = "1" ]; then
					echo "--- mariadb processlist (non-sleeping) ---"
					mysql --connect-timeout=2 -t -e \
						"SELECT ID,USER,DB,COMMAND,TIME,STATE,LEFT(INFO,300) q
						 FROM information_schema.PROCESSLIST
						 WHERE COMMAND<>'Sleep' ORDER BY TIME DESC LIMIT 40" 2>&1
					echo
					echo "--- lock waits ---"
					mysql --connect-timeout=2 -t -e \
						"SELECT ID,DB,TIME,STATE,LEFT(INFO,200) q FROM information_schema.PROCESSLIST
						 WHERE STATE LIKE '%lock%' OR STATE LIKE '%Waiting for table%' ORDER BY TIME DESC LIMIT 25" 2>&1
				fi
				echo
				echo "--- recent apache access (last 40 app requests) ---"
				for lg in /var/log/apache2/domlogs/my.evolutionerp.com.au-ssl_log \
				          /usr/local/apache/domlogs/my.evolutionerp.com.au-ssl_log; do
					[ -r "$lg" ] && tail -200 "$lg" | grep -vE '\.(css|js|png|jpe?g|gif|svg|ico|woff2?)' | tail -40 && break
				done
			} > "$I" 2>&1
			sync 2>/dev/null
		fi
	fi

	# --- retention ----------------------------------------------------------
	if [ $((tick % 360)) -eq 0 ]; then
		find "$OUTDIR" -maxdepth 1 -name 'samples-*.tsv' -mtime +"$RETAIN_DAYS" -delete 2>/dev/null
		find "$OUTDIR" -maxdepth 1 -name 'incident-*.txt' -mtime +"$RETAIN_DAYS" -delete 2>/dev/null
	fi

	sleep "$INTERVAL"
done