#!/bin/bash
###############################################################################
# start-watcher.sh — start the black-box recorder on prod
#
# Run this once. It starts forensic-watch.sh in the background, makes it
# survive both logout and reboot, verifies it is actually recording, and
# prints how to read the results.
#
#   ./start-watcher.sh                 # start (default: 10s, /var/log/erp-forensics)
#   ./start-watcher.sh -i 5            # 5-second sampling
#   ./start-watcher.sh -d /root/erpfx  # custom output directory
#   ./start-watcher.sh --no-reboot     # skip the @reboot crontab entry
#   ./start-watcher.sh --status        # is it running? what has it seen?
#   ./start-watcher.sh --stop          # stop it
#
# Idempotent: safe to run repeatedly. Read-only with respect to the system —
# it starts one sampling process and adds one crontab line, nothing else.
###############################################################################

set -u

INTERVAL=10
OUTDIR="/var/log/erp-forensics"
DO_REBOOT=1
ACTION="start"

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
WATCHER="$SCRIPT_DIR/forensic-watch.sh"

while [ $# -gt 0 ]; do
	case "$1" in
		-i|--interval) INTERVAL="$2"; shift 2 ;;
		-d|--dir)      OUTDIR="$2"; shift 2 ;;
		--no-reboot)   DO_REBOOT=0; shift ;;
		--status)      ACTION="status"; shift ;;
		--stop)        ACTION="stop"; shift ;;
		-h|--help)     sed -n '2,20p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
		*) echo "Unknown option: $1" >&2; exit 1 ;;
	esac
done

say()  { printf '%s\n' "$*"; }
ok()   { printf '  [ OK ]  %s\n' "$*"; }
warn() { printf '  [WARN]  %s\n' "$*"; }
fail() { printf '  [FAIL]  %s\n' "$*"; }

say "======================================================================"
say "ERP FORENSIC WATCHER"
say "======================================================================"

###############################################################################
# Sanity
###############################################################################

if [ ! -f "$WATCHER" ]; then
	fail "forensic-watch.sh not found next to this script."
	say  "        expected: $WATCHER"
	exit 1
fi
chmod +x "$WATCHER" 2>/dev/null

if ! bash -n "$WATCHER" 2>/dev/null; then
	fail "forensic-watch.sh has a syntax error — refusing to start."
	exit 1
fi

###############################################################################
# Output directory (fall back if the preferred one isn't writable)
###############################################################################

if ! mkdir -p "$OUTDIR" 2>/dev/null || [ ! -w "$OUTDIR" ]; then
	warn "$OUTDIR not writable — falling back to $SCRIPT_DIR/samples"
	OUTDIR="$SCRIPT_DIR/samples"
	mkdir -p "$OUTDIR" || { fail "cannot create $OUTDIR"; exit 1; }
fi

###############################################################################
# --status / --stop
###############################################################################

if [ "$ACTION" = "status" ]; then
	"$WATCHER" -d "$OUTDIR" -i "$INTERVAL" --status
	say
	say "Outage gaps detected so far:"
	"$WATCHER" -d "$OUTDIR" -i "$INTERVAL" --gaps
	exit 0
fi

if [ "$ACTION" = "stop" ]; then
	"$WATCHER" -d "$OUTDIR" --stop
	if crontab -l 2>/dev/null | grep -qF 'forensic-watch.sh'; then
		crontab -l 2>/dev/null | grep -v 'forensic-watch.sh' | crontab - 2>/dev/null \
			&& ok "removed @reboot crontab entry" \
			|| warn "could not edit crontab — remove the forensic-watch.sh line manually"
	fi
	exit 0
fi

###############################################################################
# Start
###############################################################################

say
say "Output dir : $OUTDIR"
say "Interval   : ${INTERVAL}s"
say "Watcher    : $WATCHER"
say "User       : $(id -un)"
say

[ "$(id -u)" -ne 0 ] && warn "not running as root — MariaDB processlist and some"
[ "$(id -u)" -ne 0 ] && say  "          /proc fields in incident dumps may be unavailable."

# already running?
PIDFILE="$OUTDIR/watch.pid"
if [ -r "$PIDFILE" ] && kill -0 "$(cat "$PIDFILE")" 2>/dev/null; then
	ok "already running as pid $(cat "$PIDFILE") — leaving it alone"
	say
	"$WATCHER" -d "$OUTDIR" -i "$INTERVAL" --status
	exit 0
fi
[ -e "$PIDFILE" ] && rm -f "$PIDFILE"     # stale pidfile from a hard hang

# launch, fully detached so it survives logout AND this shell dying
nohup setsid "$WATCHER" -i "$INTERVAL" -d "$OUTDIR" >/dev/null 2>&1 < /dev/null &
disown 2>/dev/null

###############################################################################
# Verify it is genuinely recording — don't just trust the fork
###############################################################################

say "Verifying..."
SAMPLEFILE="$OUTDIR/samples-$(date +%Y-%m-%d).tsv"
BEFORE=0
[ -r "$SAMPLEFILE" ] && BEFORE=$(wc -l < "$SAMPLEFILE")

WAIT=$(( INTERVAL * 2 + 5 ))
n=0
while [ $n -lt $WAIT ]; do
	sleep 1
	n=$((n+1))
	[ -r "$SAMPLEFILE" ] || continue
	AFTER=$(wc -l < "$SAMPLEFILE")
	[ "$AFTER" -gt "$BEFORE" ] && break
done

if [ -r "$PIDFILE" ] && kill -0 "$(cat "$PIDFILE")" 2>/dev/null; then
	ok "running as pid $(cat "$PIDFILE")"
else
	fail "process did not stay up. Try running it in the foreground to see why:"
	say  "        $WATCHER -i $INTERVAL -d $OUTDIR"
	exit 1
fi

if [ -r "$SAMPLEFILE" ] && [ "$(wc -l < "$SAMPLEFILE")" -gt "$BEFORE" ]; then
	ok "writing samples to $SAMPLEFILE"
else
	warn "process is up but no new sample line yet — check again in a minute:"
	say  "        tail -2 $SAMPLEFILE"
fi

# is MariaDB reachable for the processlist dumps?
if mysql --connect-timeout=2 -N -B -e "SELECT 1" >/dev/null 2>&1; then
	ok "MariaDB reachable — incident dumps will include the processlist"
else
	warn "MariaDB not reachable as $(id -un) — incident dumps will omit the"
	say  "          processlist. Fix by running as root, or set MYSQL_USER/MYSQL_PW."
fi

###############################################################################
# Survive reboot
###############################################################################

if [ "$DO_REBOOT" = "1" ]; then
	CRONLINE="@reboot $WATCHER -i $INTERVAL -d $OUTDIR"
	if crontab -l 2>/dev/null | grep -qF "forensic-watch.sh"; then
		ok "@reboot crontab entry already present"
	else
		( crontab -l 2>/dev/null; echo "$CRONLINE" ) | crontab - 2>/dev/null \
			&& ok "@reboot crontab entry installed" \
			|| warn "could not install crontab entry — add manually: $CRONLINE"
	fi
fi

###############################################################################
# Disk-use guard
###############################################################################

EST=$(awk -v i="$INTERVAL" 'BEGIN{printf "%.1f", (86400/i)*200/1048576}')
say
ok "estimated disk use: ~${EST} MB/day of samples, 14-day retention"

###############################################################################
say
say "======================================================================"
say "RUNNING. What to do next:"
say "======================================================================"
say
say "  After the next outage, run:"
say "    $SCRIPT_DIR/start-watcher.sh --status"
say
say "  That prints the peaks and, crucially, any GAP in the sample file."
say "  The gap is where the OS stopped scheduling us — the last sample before"
say "  it names the failure mode, and any incident-*.txt dump names the"
say "  process that ate the memory."
say
say "  Files to send over:"
say "    $OUTDIR/samples-*.tsv"
say "    $OUTDIR/incident-*.txt"
say
say "  Stop with:  $SCRIPT_DIR/start-watcher.sh --stop"
say