#!/usr/bin/env bash
#
# prod-log-digest.sh — turn the synced production logs into a bounded report.
#
# sync-prod-logs.sh runs this automatically; it is also useful standalone
# against an already-synced tree:
#
#   ./prod-log-digest.sh                        > /tmp/digest.md
#   ./prod-log-digest.sh --since 30 --top 40
#   ./prod-log-digest.sh --data /path/to/projects/production
#
# The point is that a few hundred MB of log is unreadable. Every section here
# is grouped, counted and capped, so the whole digest stays small enough to
# read in one sitting while still naming the file and line to go and fix.
#
# NOTE: deliberately no `pipefail`. Nearly every pipeline here ends in `head`,
# which closes the pipe and SIGPIPEs the producer; under pipefail that reads as
# failure and every truncated section would report itself empty.
set -u

# Byte-wise sort/grep. Under a UTF-8 locale every pass over the web error log
# (834MB on first pull) is several times slower, and the digest crawls.
export LC_ALL=C

DATA="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/projects/production"
TOP=25
SINCE_DAYS=14

while [[ $# -gt 0 ]]; do
	case "$1" in
		--data)  DATA="$2"; shift 2 ;;
		--top)   TOP="$2"; shift 2 ;;
		--since) SINCE_DAYS="$2"; shift 2 ;;
		-h|--help) sed -n '2,16p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
		*) echo "unknown option: $1" >&2; exit 2 ;;
	esac
done

LOGS="$DATA/logs"
[[ -d "$LOGS" ]] || { echo "no synced logs at $LOGS — run sync-prod-logs.sh first" >&2; exit 1; }

# Read a log family whether or not it has been rotated/compressed.
catlogs() {
	local found=0 f
	for f in "$@"; do
		[[ -f "$f" ]] || continue
		found=1
		case "$f" in
			*.gz) zcat -f -- "$f" 2>/dev/null ;;
			*)    cat -- "$f" 2>/dev/null ;;
		esac
	done
	return $((1 - found))
}

# Concatenate a log family oldest-first. Glob order is wrong for this: it puts
# error_log before error_log.1.gz, i.e. NEWEST first, which silently inverts
# "first/last entry" and makes a `tail` of "most recent" entries return the
# oldest rotation instead.
catlogs_ordered() {
	local files=()
	mapfile -t files < <(ls -tr -- "$@" 2>/dev/null)
	[[ ${#files[@]} -gt 0 ]] || return 1
	catlogs "${files[@]}"
}

h1() { printf '\n## %s\n\n' "$*"; }
h2() { printf '\n### %s\n\n' "$*"; }
none() { printf '_none_\n'; }

# Indent a section's output, or say so when there is nothing. Checking the
# captured text is the only reliable emptiness test: a grep that matches
# nothing exits 1, but a grep that matches nothing *through a pipeline* often
# exits 0, so neither exit status nor `|| none` can be trusted.
emit() {
	local out; out="$(cat)"
	if [[ -z "${out//[[:space:]]/}" ]]; then none; else printf '%s\n' "$out" | sed 's/^/    /'; fi
}

# ---------------------------------------------------------------------------
# Normalise a PHP error line so the same bug collapses to a single row.
#
# Both log formats are handled: PHP's own error_log
#   [24-Jul-2026 19:31:44 Australia/Sydney] PHP Fatal error: ...
# and Apache's, which prod's vhost log also uses
#   [Fri Jul 24 19:31:44 2026] [php:error] [pid 123] [client 1.2.3.4:5] PHP ...
#
# What has to go, or one bug reports as four:
#   - every leading [bracketed] field (timestamp, pid, and especially client,
#     whose IP and ephemeral port differ on every single occurrence)
#   - the ", referer: <url>" tail — a different URL per hit
#   - the inline stack trace: Apache escapes the newlines, so the whole trace
#     with its per-request arguments rides along on one line
#   - digits: line numbers, ids, byte counts
# ---------------------------------------------------------------------------
normalise() {
	sed -E \
		-e 's/^(\[[^]]*\][[:space:]]*)+//' \
		-e 's/(\\n|, )?Stack trace:.*$//' \
		-e 's/,[[:space:]]*referer:.*$//' \
		-e 's/\b0x[0-9a-fA-F]+\b/0xH/g' \
		-e 's/[0-9]+/N/g' \
		-e 's/[[:space:]]+/ /g' \
		-e 's/^ //; s/ $//'
}

# Group normalised lines: "<count>  <message>", most frequent first.
group() {
	normalise | sort | uniq -c | sort -rn | head -"$TOP" \
		| awk -v w=220 '{c=$1; $1=""; sub(/^ /,""); printf "%6d  %s\n", c, substr($0,1,w)}'
}

printf '# Production log digest\n\n'
printf -- '- generated: %s\n' "$(date '+%Y-%m-%d %H:%M:%S %z')"
[[ -f "$DATA/state/last-sync.txt" ]] && printf -- '- last sync: %s\n' "$(cat "$DATA/state/last-sync.txt")"
printf -- '- mirror: `%s` (%s)\n' "$LOGS" "$(du -sh "$LOGS" 2>/dev/null | cut -f1)"
printf -- '- window for dated sources: last %s days\n' "$SINCE_DAYS"

# ---------------------------------------------------------------------------
h1 "Source status"
if [[ -f "$DATA/state/sources.tsv" ]]; then
	printf '| source | status | size on prod |\n|---|---|---|\n'
	awk -F'\t' '{
		sz = $3 + 0; unit = "B"
		if (sz > 1073741824)   { sz = sz/1073741824; unit="GB" }
		else if (sz > 1048576) { sz = sz/1048576; unit="MB" }
		else if (sz > 1024)    { sz = sz/1024; unit="KB" }
		printf "| %s | %s | %.1f %s |\n", $1, $2, sz, unit
	}' "$DATA/state/sources.tsv"
	printf '\nDENIED means the path exists but the sync account cannot read it — '
	printf 'set `PROD_USER=root` or `PROD_SUDO=1` in the credentials file.\n'
else
	none
fi

# ---------------------------------------------------------------------------
# WEB ERROR LOG — the main event
# ---------------------------------------------------------------------------
h1 "ERP error log"
TMP="$(mktemp)"; trap 'rm -f "$TMP"' EXIT
if ! catlogs_ordered "$LOGS"/web/error_log* > "$TMP" 2>/dev/null || [[ ! -s "$TMP" ]]; then
	none
else
	printf -- '- lines: %s\n' "$(wc -l < "$TMP")"
	printf -- '- first entry: %s\n' "$(grep -m1 -oE '^\[[^]]*\]' "$TMP" | tr -d '[]')"
	printf -- '- last entry: %s\n'  "$(grep -oE '^\[[^]]*\]' "$TMP" | tail -1 | tr -d '[]')"

	h2 "By severity"
	printf '| severity | count |\n|---|---|\n'
	grep -oE 'PHP (Fatal error|Parse error|Recoverable fatal error|Warning|Deprecated|Notice|Startup)' "$TMP" \
		| sort | uniq -c | sort -rn \
		| awk '{c=$1; $1=""; sub(/^ /,""); printf "| %s | %s |\n", $0, c}'

	# Fatals first: every one of these is a 500 in a user's face.
	h2 "Fatal / parse errors (top $TOP)"
	grep -E 'PHP (Fatal error|Parse error|Recoverable fatal error)' "$TMP" | group | emit

	h2 "Most recent fatals (real timestamps and line numbers)"
	grep -E 'PHP (Fatal error|Parse error)' "$TMP" | tail -12 | cut -c1-260 | emit

	# Prod moved to PHP 8.3 on 03-Sep-2026, so the live risk is runtime
	# TypeErrors and deprecations rather than anything that fails to parse.
	h2 "Deprecations and warnings (top $TOP) — PHP 8.3 migration surface"
	grep -E 'PHP (Deprecated|Warning)' "$TMP" | group | emit

	h2 "Memory exhaustion and timeouts, by script"
	grep -E 'Allowed memory size|Maximum execution time' "$TMP" \
		| grep -oE '/(home|var/www)/[^ :]+\.(php|inc)' | sort | uniq -c | sort -rn | head -"$TOP" | emit

	h2 "Noisiest files overall (top $TOP)"
	grep -oE '/(home|var/www)/[^ :()]+\.(php|inc)' "$TMP" \
		| sort | uniq -c | sort -rn | head -"$TOP" | emit
fi

# ---------------------------------------------------------------------------
h1 "Tenant plugin error logs"
# Pulled with rsync -R, so each tenant gets its own directory and two tenants
# cannot overwrite each other's identically-named error_log.
mapfile -t TENANT_LOGS < <(find "$LOGS/tenant" -type f 2>/dev/null | sort)
if [[ ${#TENANT_LOGS[@]} -eq 0 ]]; then
	none
else
	for f in "${TENANT_LOGS[@]}"; do
		rel="${f#"$LOGS"/tenant/}"
		printf -- '- `%s` — %s lines, last written %s\n' \
			"$rel" "$(wc -l < "$f")" "$(date -r "$f" '+%Y-%m-%d %H:%M' 2>/dev/null)"
	done
	h2 "Grouped across all tenants (top $TOP)"
	catlogs "${TENANT_LOGS[@]}" | grep -E 'PHP |Error|error' | group | emit
fi

# ---------------------------------------------------------------------------
h1 "Cron"
if compgen -G "$LOGS/cron/*" > /dev/null 2>&1; then
	for f in "$LOGS"/cron/*; do
		[[ -f "$f" ]] || continue
		printf -- '- `%s` — %s lines, last written %s\n' \
			"$(basename "$f")" "$(wc -l < "$f" 2>/dev/null)" \
			"$(date -r "$f" '+%Y-%m-%d %H:%M' 2>/dev/null)"
	done

	h2 "Failures (top $TOP)"
	catlogs "$LOGS"/cron/* \
		| grep -iE 'error|fatal|exception|failed|failure|denied|refused|timeout|deadlock|could not|unable to' \
		| group | emit

	h2 "Tail of each cron log"
	for f in "$LOGS"/cron/*; do
		[[ -f "$f" ]] || continue
		printf '  %s:\n' "$(basename "$f")"
		tail -8 "$f" 2>/dev/null | cut -c1-200 | sed 's/^/      /'
		printf '\n'
	done
else
	none
fi

# ---------------------------------------------------------------------------
h1 "Deploys"
catlogs "$LOGS"/deploy/* 2>/dev/null \
	| grep -E 'deploying|deploy complete|ABORT|WARNING' | tail -25 | cut -c1-200 | emit

# ---------------------------------------------------------------------------
h1 "MariaDB"
MY="$(mktemp)"
if ! catlogs_ordered "$LOGS"/mysql/* > "$MY" 2>/dev/null || [[ ! -s "$MY" ]]; then
	none
else
	h2 "Crash / corruption signatures"
	grep -E 'mysqld got signal|Assertion failure|was not shut down normally|marked as crashed|Out of memory|cannot allocate|Too many connections' "$MY" \
		| tail -25 | cut -c1-220 | emit

	h2 "Errors grouped (top $TOP)"
	grep -F '[ERROR]' "$MY" | group | emit

	h2 "Counters"
	{
		printf 'aborted connections:        %s\n' "$(grep -cE 'Aborted connection' "$MY")"
		printf 'lock timeouts + deadlocks:  %s\n' "$(grep -cE 'Lock wait timeout|Deadlock found' "$MY")"
		printf 'table crash repairs:        %s\n' "$(grep -cE 'marked as crashed|Checking table' "$MY")"
	} | emit

	h2 "Restarts"
	grep -E 'ready for connections|Starting MariaDB|Shutdown complete|Normal shutdown' "$MY" \
		| tail -15 | cut -c1-200 | emit

	if compgen -G "$LOGS/mysql/*slow*" > /dev/null 2>&1; then
		h2 "Slowest queries (top 10 by Query_time)"
		catlogs "$LOGS"/mysql/*slow* \
			| awk '/^# Query_time/ { qt=$3 }
			       tolower($0) ~ /^(select|insert|update|delete)/ && qt != "" {
			           printf "%8.2fs  %s\n", qt, substr($0,1,170); qt="" }' \
			| sort -rn | head -10 | emit
	fi
fi
rm -f "$MY"

# ---------------------------------------------------------------------------
h1 "Kernel / OOM"
SYS="$(mktemp)"
if ! catlogs_ordered "$LOGS"/system/messages* > "$SYS" 2>/dev/null || [[ ! -s "$SYS" ]]; then
	none
else
	h2 "OOM kills"
	grep -E 'Out of memory|oom-kill|oom_reaper|Killed process' "$SYS" | tail -30 | cut -c1-220 | emit

	h2 "Victims ranked"
	grep -oE '(Killed process [0-9]+ \(|oom-kill:.*task=)[^ ,)]+' "$SYS" \
		| sed -E 's/.*\(//; s/.*task=//' | sort | uniq -c | sort -rn | head -15 | emit

	h2 "Hangs, IO errors, segfaults"
	grep -E 'hung_task|blocked for more than|I/O error|EXT4-fs error|kernel: BUG|segfault' "$SYS" \
		| tail -20 | cut -c1-200 | emit
fi
rm -f "$SYS"

# ---------------------------------------------------------------------------
# FORENSIC RECORDER
#
# The outages take the whole OS down, SSH included, so the recorder's own
# scheduling gap is the primary signal — the classification table is in
# projects/OOM and Server Load Forensic/README.md.
#
# Columns: 1 iso, 2 epoch, 3 load1, 6 blocked(D), 8 mem_avail_mb,
#   9 mem_avail_pct, 11 swap_used_mb, 13 swapout_s, 14 iowait_pct, 15 steal_pct,
#   17 mysql_conn, 18 mysqld_rss_mb, 19 php_procs, 20 php_rss_mb,
#   21 apache_procs, 23 tcp_estab, 24 php_top_user, 26 php_top_rss_mb
# ---------------------------------------------------------------------------
h1 "Forensic recorder"
if compgen -G "$LOGS/forensics/samples-*.tsv" > /dev/null 2>&1; then
	SAMP="$(mktemp)"
	# Two watcher instances ran 08-12 -> 08-27, so every sample in that range
	# was recorded twice. Dedupe on epoch before counting anything.
	cat "$LOGS"/forensics/samples-*.tsv 2>/dev/null \
		| grep -v $'^iso\tepoch' \
		| awk -F'\t' 'NF>20 && $2 ~ /^[0-9]+$/ && !seen[$2]++' \
		| sort -t$'\t' -k2,2n > "$SAMP"

	printf -- '- samples: %s (deduped) across %s day file(s)\n' \
		"$(wc -l < "$SAMP")" "$(ls -1 "$LOGS"/forensics/samples-*.tsv 2>/dev/null | wc -l)"
	printf -- '- window: %s -> %s\n' \
		"$(head -1 "$SAMP" | cut -f1)" "$(tail -1 "$SAMP" | cut -f1)"

	h2 "Scheduling gaps (candidate outages)"
	printf 'A gap is where the OS stopped scheduling the recorder. The last sample\n'
	printf 'before it names the failure mode: memory collapse = the application or DB;\n'
	printf 'blocked(D) spiking with memory fine = storage stall; high steal%% = the\n'
	printf 'hypervisor; no degradation at all = a host event, not the ERP.\n\n'
	awk -F'\t' -v cap=15 '
		NR>1 && ($2 - pe) > 40 {
			n++
			mins = ($2-pe)/60
			if (mins > worst) { worst = mins; worst_at = $1 }
			if (n <= cap) {
				printf "GAP %5.1f min, ended %s\n", mins, $1
				printf "    last sample before: %s\n", pi
				printf "    load1=%s blocked(D)=%s mem_avail=%sMB/%s%% swap_used=%sMB swapout/s=%s iowait=%s%% steal=%s%%\n", p3,p6,p8,p9,p11,p13,p14,p15
				printf "    mysql_conn=%s mysqld_rss=%sMB php_procs=%s php_rss=%sMB apache=%s estab=%s top_user=%s\n\n", p17,p18,p19,p20,p21,p23,p24
			}
		}
		{ pe=$2; pi=$1; p3=$3; p6=$6; p8=$8; p9=$9; p11=$11; p13=$13; p14=$14; p15=$15
		  p17=$17; p18=$18; p19=$19; p20=$20; p21=$21; p23=$23; p24=$24 }
		END {
			if (!n) { print "none - the recorder has not missed a beat in this window"; exit }
			if (n > cap) printf "... %d more gap(s) not shown\n\n", n - cap
			printf "%d gap(s) total; longest %.1f min ending %s\n", n, worst, worst_at
		}
	' "$SAMP" | emit

	h2 "Peaks in window"
	awk -F'\t' '
		BEGIN { prss=pproc=pconn=pmy=pswap=pload=psteal=pblk=-1 }
		{ if ($20+0 > prss)  { prss=$20+0;  prss_t=$1; prss_u=$24 }
		  if ($19+0 > pproc) { pproc=$19+0; pproc_t=$1 }
		  if ($17+0 > pconn) { pconn=$17+0; pconn_t=$1 }
		  if ($18+0 > pmy)   { pmy=$18+0;   pmy_t=$1 }
		  if ($11+0 > pswap) { pswap=$11+0; pswap_t=$1 }
		  if ($3+0  > pload) { pload=$3+0;  pload_t=$1 }
		  if ($15+0 > psteal){ psteal=$15+0; psteal_t=$1 }
		  if ($6+0  > pblk)  { pblk=$6+0;   pblk_t=$1 }
		  if (mav=="" || $8+0 < mav) { mav=$8+0; mav_t=$1 } }
		END {
			printf "peak php_rss     %9s MB  at %s   (top user: %s)\n", prss, prss_t, prss_u
			printf "peak php_procs   %9s     at %s\n", pproc, pproc_t
			printf "peak mysql_conn  %9s     at %s\n", pconn, pconn_t
			printf "peak mysqld_rss  %9s MB  at %s\n", pmy, pmy_t
			printf "peak swap_used   %9s MB  at %s\n", pswap, pswap_t
			printf "peak load1       %9s     at %s\n", pload, pload_t
			printf "peak blocked(D)  %9s     at %s\n", pblk, pblk_t
			printf "peak steal%%      %9s     at %s\n", psteal, psteal_t
			printf "min  mem_avail   %9s MB  at %s\n", mav, mav_t
		}
	' "$SAMP" | emit

	h2 "Threshold breaches per day (v2.1 trips on php_procs>=25, php_rss>=1500MB, mysql_conn>=15)"
	# Listed per day with that day's worst sample. A flat chronological list is
	# no use here: mysql_conn>=15 trips almost continuously on this box (normal
	# running is 90-110), so the first 30 rows would only ever be day one.
	#
	# Sorting is done by sort(1), not in awk: this box has mawk, which has no
	# asorti(). The day is the leading ISO field, so a plain sort is correct.
	{
		printf '%-12s %8s   %s\n' "day" "samples" "worst sample that day (by php_rss)"
		awk -F'\t' '
			$19+0 >= 25 || $20+0 >= 1500 || $17+0 >= 15 {
				day = substr($1, 1, 10)
				c[day]++
				if ($20+0 > wrss[day]) {
					wrss[day] = $20+0
					worst[day] = sprintf("%s php_procs=%s php_rss=%sMB mysql_conn=%s mem_avail=%s%% top_user=%s",
					                     substr($1,12), $19, $20, $17, $9, $24)
				}
			}
			END { for (d in c) printf "%-12s %8d   %s\n", d, c[d], worst[d] }
		' "$SAMP" | sort
	} | emit

	h2 "Busiest PHP accounts (samples where each held the top slot)"
	awk -F'\t' '$24 != "" && $24 != "-" {c[$24]++} END {for (u in c) printf "%8d  %s\n", c[u], u}' "$SAMP" \
		| sort -rn | head -15 | emit
	rm -f "$SAMP"
else
	printf '_No sample files. Is the recorder still running on prod?_ `pgrep -fa forensic-watch.sh`\n'
fi

h2 "Incident dumps"
if compgen -G "$LOGS/forensics/incident-*.txt" > /dev/null 2>&1; then
	printf -- '- %s dump(s), newest last:\n\n' "$(ls -1 "$LOGS"/forensics/incident-*.txt | wc -l)"
	ls -1t "$LOGS"/forensics/incident-*.txt | head -8 | tac | while read -r f; do
		printf '%s  (%s)\n' "$(basename "$f")" "$(du -h "$f" | cut -f1)"
		grep -m3 -E 'TRIGGER|php_procs=|php_rss=|mysql_conn=' "$f" 2>/dev/null | cut -c1-170 | sed 's/^/    /'
	done | emit
	printf '\nRead the newest in full. The tables that answer *which URL* are '
	printf '**top request paths across all vhosts** and **backup / dump / scan processes**.\n'
else
	none
fi

# ---------------------------------------------------------------------------
h1 "Apache per-vhost logs"
if compgen -G "$LOGS/domlogs/*" > /dev/null 2>&1; then
	h2 "Busiest vhosts in the synced window"
	for f in "$LOGS"/domlogs/*; do
		[[ -f "$f" ]] || continue
		printf '%8d  %s\n' "$(wc -l < "$f")" "$(basename "$f")"
	done | sort -rn | head -15 | emit

	h2 "5xx responses by status and path (top $TOP)"
	catlogs "$LOGS"/domlogs/* | awk '$9 ~ /^5[0-9][0-9]$/ {print $9, $7}' \
		| sort | uniq -c | sort -rn | head -"$TOP" | cut -c1-200 | emit
else
	printf '_Not synced (opt-in). Run `sync-prod-logs.sh --domlogs` when you need request-level detail._\n'
fi

printf '\n---\n\n'
printf 'Refresh: `prod-tools/sync-prod-logs.sh`  ·  full run log: `%s`\n' "$DATA/state/sync.log"