#!/bin/bash
###############################################################################
# forensic-web.sh — Evolution ERP Apache access/error log forensics
#
# Supersedes erp_forensic.sh + erp_forensic_oom.sh (same job, correct parsing).
#
# WHAT IT FIXES vs the old scripts
#   * page= is extracted from the REQUEST URI only. The old scripts used
#     `sed 's/.*page=\(...\)/\1/'` which is greedy and matched the LAST page=
#     on the line — i.e. the REFERER. Heartbeat POSTs (163 of 511 requests in
#     the 10-Jul sample) were therefore counted as page loads of whatever
#     screen the user happened to have open. Referer-derived screens are still
#     reported, but in their own clearly-labelled section.
#   * Time window is parsed from the timestamp field and compared numerically.
#     The old START_REGEX "10:[00-10]:" is a character class, not a range —
#     it matched (almost) nothing.
#   * Log file is auto-discovered across cPanel/EA4 layouts instead of being
#     hard-coded to two different wrong paths in two different scripts.
#   * The log is parsed ONCE into a temp TSV; every section is a cheap sort on
#     that instead of 15 sequential greps over a multi-hundred-MB log.
#   * Status codes, response bytes, retry amplification, per-minute request
#     AND distinct-IP histograms, and the Apache error log (PHP OOM / CGI
#     timeouts) are all covered — none of which the old scripts looked at.
#   * Default output is a capped digest small enough to paste into a Claude
#     session. --full removes the caps.
#
# USAGE
#   ./forensic-web.sh                      # whole of today; finds the spikes
#   ./forensic-web.sh -f 10:00 -t 10:20    # zoom into a window
#   ./forensic-web.sh -d 26/Jul/2026 -f 14:00 -t 14:30
#   ./forensic-web.sh -l /path/to/log -e /path/to/error_log --full
#
# Read-only. Safe to run at any time, including during an incident.
###############################################################################

DATE_DEFAULT="$(date +%d/%b/%Y)"
DATE="$DATE_DEFAULT"
FROM="0000"
TO="2359"
LOG=""
ERRLOG=""
FULL=0
OUT=""

usage() {
	sed -n '2,40p' "$0" | sed 's/^# \{0,1\}//'
	exit 1
}

hm() { echo "$1" | tr -d ':' ; }

while [ $# -gt 0 ]; do
	case "$1" in
		-d|--date)   DATE="$2"; shift 2 ;;
		-f|--from)   FROM="$(hm "$2")"; shift 2 ;;
		-t|--to)     TO="$(hm "$2")"; shift 2 ;;
		-l|--log)    LOG="$2"; shift 2 ;;
		-e|--error)  ERRLOG="$2"; shift 2 ;;
		-o|--out)    OUT="$2"; shift 2 ;;
		--full)      FULL=1; shift ;;
		-h|--help)   usage ;;
		*) echo "Unknown option: $1" >&2; usage ;;
	esac
done

###############################################################################
# Log discovery
###############################################################################

if [ -z "$LOG" ]; then
	for c in \
		/var/log/apache2/domlogs/my.evolutionerp.com.au-ssl_log \
		/var/log/apache2/domlogs/*/my.evolutionerp.com.au-ssl_log \
		/usr/local/apache/domlogs/my.evolutionerp.com.au-ssl_log \
		/usr/local/apache/domlogs/*/my.evolutionerp.com.au-ssl_log \
		/etc/apache2/logs/domlogs/my.evolutionerp.com.au-ssl_log \
		/home/*/logs/my.evolutionerp.com.au-ssl_log
	do
		[ -r "$c" ] && LOG="$c" && break
	done
fi

if [ -z "$LOG" ] || [ ! -r "$LOG" ]; then
	echo "ERROR: could not find a readable ERP access log." >&2
	echo "Tried the usual cPanel/EA4 domlog paths. Pass one with -l." >&2
	echo "Hint: ls -la /var/log/apache2/domlogs/ | grep evolution" >&2
	exit 2
fi

if [ -z "$ERRLOG" ]; then
	for c in \
		/var/log/apache2/error_log \
		/usr/local/apache/logs/error_log \
		/var/log/apache2/error.log \
		/etc/apache2/logs/error_log
	do
		[ -r "$c" ] && ERRLOG="$c" && break
	done
fi

CAP_SM=15; CAP_MD=25; CAP_LG=40
if [ "$FULL" = "1" ]; then CAP_SM=60; CAP_MD=150; CAP_LG=500; fi

STAMP="$(echo "$DATE" | tr '/' '-')_${FROM}-${TO}"
[ -z "$OUT" ] && OUT="forensic-web_${STAMP}.txt"

TMP="$(mktemp -d /tmp/erpforensic.XXXXXX)" || exit 3
trap 'rm -rf "$TMP"' EXIT INT TERM

exec > "$OUT" 2>&1

hr()  { echo "======================================================================"; }
sec() { echo; hr; echo "$1"; hr; }

echo "EVOLUTION ERP — WEB FORENSICS"
hr
echo "Access log : $LOG"
echo "Error log  : ${ERRLOG:-<not found>}"
echo "Date       : $DATE"
echo "Window     : ${FROM:0:2}:${FROM:2:2} — ${TO:0:2}:${TO:2:2}"
echo "Mode       : $([ "$FULL" = 1 ] && echo full || echo digest)"
echo "Generated  : $(date '+%Y-%m-%d %H:%M:%S %Z') on $(hostname)"

###############################################################################
# Single parse pass -> TSV
# cols: 1 hhmm  2 ip  3 method  4 uri  5 status  6 bytes  7 reqpage
#       8 refpage  9 uaclass  10 endpoint(normalised)
###############################################################################

awk -v DATE="$DATE" -v FROM="$FROM" -v TO="$TO" '
function param(s, key,   m, v) {
	if (match(s, "[?&]" key "=[^& \"]*")) {
		v = substr(s, RSTART, RLENGTH)
		sub("^[?&]" key "=", "", v)
		return v
	}
	return ""
}
{
	i = index($0, "[")
	if (i == 0) next
	ts = substr($0, i+1, 20)
	if (substr(ts, 1, 11) != DATE) next
	hhmm = substr(ts, 13, 2) substr(ts, 16, 2)
	if (hhmm < FROM || hhmm > TO) next

	n = split($0, q, "\"")
	req = (n >= 2 ? q[2] : "")
	ref = (n >= 4 ? q[4] : "")
	ua  = (n >= 6 ? q[6] : "")

	split(req, r, " ")
	method = r[1]; uri = r[2]
	if (uri == "") uri = "-"

	split(q[3], s, " ")
	status = (s[1] == "" ? "-" : s[1])
	bytes  = (s[2] ~ /^[0-9]+$/ ? s[2] : 0)

	reqpage = param(uri, "page")
	refpage = param(ref, "page")
	if (reqpage == "") reqpage = "-"
	if (refpage == "") refpage = "-"

	uaclass = "human"
	if (ua ~ /(bot|Bot|crawl|Crawl|spider|Spider|slurp|scan|Scan|curl|Wget|python|Python|Go-http|libwww|HeadlessChrome)/)
		uaclass = "bot"

	# normalised endpoint: path + ?call=..&page=.. with numeric ids collapsed
	ep = uri
	sub(/#.*/, "", ep)
	split(ep, e, "?")
	path = e[1]
	call = param(uri, "call")
	pg   = (reqpage == "-" ? "" : reqpage)
	ep = path
	if (pg   != "") ep = ep "?page=" pg
	if (call != "") ep = ep (pg != "" ? "&" : "?") "call=" call

	printf "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", \
		hhmm, $1, method, uri, status, bytes, reqpage, refpage, uaclass, ep
}' "$LOG" > "$TMP/rec.tsv"

TOTAL=$(wc -l < "$TMP/rec.tsv")

if [ "$TOTAL" -eq 0 ]; then
	echo
	echo "!! No requests matched. Check the date format (must be dd/Mon/yyyy, e.g. 27/Jul/2026)"
	echo "   and that the window is right. Sample line from the log:"
	head -1 "$LOG"
	exit 0
fi

# non-asset view
grep -vE $'\t'"[^\t]*\.(css|js|png|jpe?g|gif|svg|ico|woff2?|ttf|eot|map)(\?[^\t]*)?"$'\t' "$TMP/rec.tsv" \
	> "$TMP/app.tsv" || true
APPTOTAL=$(wc -l < "$TMP/app.tsv")

sec "1. TRAFFIC SHAPE"
echo "Total requests in window : $TOTAL"
echo "Application requests     : $APPTOTAL  (assets/static excluded)"
echo "Distinct client IPs      : $(cut -f2 "$TMP/rec.tsv" | sort -u | wc -l)"
echo
echo "Busiest minutes (requests | distinct IPs):"
cut -f1,2 "$TMP/rec.tsv" | sort | uniq | cut -f1 | sort | uniq -c \
	| awk '{print $2"\t"$1}' | sort > "$TMP/ipmin.tsv"
cut -f1 "$TMP/rec.tsv" | sort | uniq -c | awk '{print $2"\t"$1}' | sort > "$TMP/reqmin.tsv"
join -t $'\t' "$TMP/reqmin.tsv" "$TMP/ipmin.tsv" \
	| sort -t $'\t' -k2,2nr | head -$CAP_MD \
	| awk -F'\t' '
		{ m[NR]=$1; r[NR]=$2; p[NR]=$3; if ($2+0 > mx) mx=$2+0; n=NR }
		END {
			for (i=1; i<=n; i++) {
				w = int(r[i]*45/mx); if (w<1) w=1
				bar=""; for (j=0;j<w;j++) bar=bar "#"
				printf "  %s:%s  %6d req  %4d ip  %s\n", substr(m[i],1,2), substr(m[i],3,2), r[i], p[i], bar
			}
		}'
echo
echo "Hourly totals:"
cut -c1-2 "$TMP/rec.tsv" | sort | uniq -c | awk '{printf "  %s:00  %6d\n", $2, $1}'

sec "2. HTTP STATUS / ERRORS"
cut -f5 "$TMP/rec.tsv" | sort | uniq -c | sort -nr | awk '{printf "  %-6s %6d\n", $2, $1}'
echo
FIVEXX=$(awk -F'\t' '$5 ~ /^5/' "$TMP/rec.tsv" | wc -l)
echo "5xx responses: $FIVEXX"
if [ "$FIVEXX" -gt 0 ]; then
	echo
	echo "  5xx by endpoint:"
	awk -F'\t' '$5 ~ /^5/ {print $5"  "$10}' "$TMP/rec.tsv" | sort | uniq -c | sort -nr | head -$CAP_MD \
		| awk '{printf "  %6d  %s\n", $1, substr($0, index($0,$2))}'
	echo
	echo "  5xx by minute:"
	awk -F'\t' '$5 ~ /^5/ {print substr($1,1,2)":"substr($1,3,2)}' "$TMP/rec.tsv" \
		| sort | uniq -c | sort -k2 | awk '{printf "  %s  %4d\n", $2, $1}'
fi

sec "3. REAL PAGE LOADS  (page= parsed from the REQUEST URI only)"
echo "This is the number that matters. The old scripts reported the referer."
echo
awk -F'\t' '$7 != "-" && $4 ~ /index\.php/ {print $7}' "$TMP/rec.tsv" \
	| sort | uniq -c | sort -nr | head -$CAP_MD | awk '{printf "  %6d  %s\n", $1, $2}'
echo
echo "  (any request carrying page= in its own URI, incl. AJAX handlers):"
awk -F'\t' '$7 != "-" {print $7}' "$TMP/rec.tsv" \
	| sort | uniq -c | sort -nr | head -$CAP_MD | awk '{printf "  %6d  %s\n", $1, $2}'

sec "4. SCREENS USERS HAD OPEN  (page= from the REFERER — concurrency proxy)"
echo "requests | distinct IPs — a high IP count = that many people sitting on it."
echo
awk -F'\t' '$8 != "-" {print $8"\t"$2}' "$TMP/rec.tsv" | sort -u | cut -f1 | sort | uniq -c \
	| awk '{print $2"\t"$1}' | sort > "$TMP/refip.tsv"
awk -F'\t' '$8 != "-" {print $8}' "$TMP/rec.tsv" | sort | uniq -c \
	| awk '{print $2"\t"$1}' | sort > "$TMP/refreq.tsv"
join -t $'\t' "$TMP/refreq.tsv" "$TMP/refip.tsv" | sort -t $'\t' -k2,2nr | head -$CAP_MD \
	| awk -F'\t' '{printf "  %6d req  %4d ip  %s\n", $2, $3, $1}'

sec "5. TOP APPLICATION ENDPOINTS  (assets excluded, ids collapsed)"
cut -f10 "$TMP/app.tsv" | sort | uniq -c | sort -nr | head -$CAP_LG \
	| awk '{printf "  %6d  %s\n", $1, $2}'

sec "6. EXPENSIVE ENDPOINT CLASSES"
echo "-- reports / exports / print / PDF:"
grep -iE 'report|export|csv|excel|print|pdf|download' "$TMP/app.tsv" \
	| cut -f10 | sort | uniq -c | sort -nr | head -$CAP_MD | awk '{printf "  %6d  %s\n", $1, $2}'
echo
echo "-- register/grid filter handlers (SQL_CALC_FOUND_ROWS + LIKE scans):"
grep -E 'call=(filter|fetch|get)' "$TMP/app.tsv" \
	| cut -f10 | sort | uniq -c | sort -nr | head -$CAP_MD | awk '{printf "  %6d  %s\n", $1, $2}'
echo
echo "-- global search:"
grep -iE 'globalsearch' "$TMP/app.tsv" | cut -f10 | sort | uniq -c | sort -nr | head -$CAP_SM \
	| awk '{printf "  %6d  %s\n", $1, $2}'
echo
echo "-- cron / plugin entry points:"
grep -iE '/(cron|plugins)/' "$TMP/app.tsv" \
	| cut -f10 | sort | uniq -c | sort -nr | head -$CAP_MD | awk '{printf "  %6d  %s\n", $1, $2}'

sec "7. RETRY AMPLIFICATION  (same IP hitting the same endpoint repeatedly)"
echo "CGI timeout -> user reloads -> death spiral. >=5 in the window is suspicious."
echo
awk -F'\t' '{print $2"\t"$10}' "$TMP/app.tsv" | sort | uniq -c | sort -nr \
	| awk '$1 >= 5' | head -$CAP_MD | awk '{printf "  %6d  %s\n", $1, substr($0, index($0,$2))}'

sec "8. PER-DOCUMENT-ID BREAKDOWN  (retry storm vs many distinct records)"
for pg in jobedit quotedit invedit purchaseadd dispatchedit inventory_picking pickingedit; do
	c=$(awk -F'\t' -v p="$pg" '$7 == p' "$TMP/rec.tsv" | wc -l)
	[ "$c" -eq 0 ] && continue
	echo "-- $pg ($c requests carrying page=$pg)"
	awk -F'\t' -v p="$pg" '$7 == p {
		id="-"
		if (match($4, /[?&](id|jobid|quoteid|purchaseid|dispatchid|itemid)=[0-9]+/)) {
			id = substr($4, RSTART, RLENGTH); sub(/^[?&]/, "", id)
		}
		print id
	}' "$TMP/rec.tsv" | sort | uniq -c | sort -nr | head -$CAP_SM | awk '{printf "     %5d  %s\n", $1, $2}'
	echo
done

sec "9. HEAVIEST RESPONSES BY BYTES  (big buffered result sets)"
sort -t $'\t' -k6,6nr "$TMP/app.tsv" | head -$CAP_MD \
	| awk -F'\t' '{printf "  %10.1f KB  %s:%s  %s  %s\n", $6/1024, substr($1,1,2), substr($1,3,2), $5, substr($4,1,110)}'
echo
echo "Total bytes served in window: $(awk -F'\t' '{s+=$6} END {printf "%.1f MB", s/1048576}' "$TMP/rec.tsv")"

sec "10. TOP CLIENT IPs"
cut -f2 "$TMP/rec.tsv" | sort | uniq -c | sort -nr | head -$CAP_MD \
	| awk '{printf "  %6d  %s\n", $1, $2}'
echo
echo "Bot/scanner traffic: $(awk -F'\t' '$9=="bot"' "$TMP/rec.tsv" | wc -l) requests"
awk -F'\t' '$9=="bot" {print $2"  "$10}' "$TMP/rec.tsv" | sort | uniq -c | sort -nr | head -$CAP_SM \
	| awk '{printf "  %6d  %s\n", $1, substr($0, index($0,$2))}'

###############################################################################
# Error log — PHP fatals / memory exhaustion / CGI timeouts
###############################################################################

sec "11. APACHE ERROR LOG  (PHP OOM, CGI timeouts, fatals)"
if [ -z "$ERRLOG" ] || [ ! -r "$ERRLOG" ]; then
	echo "  Not readable — pass one with -e, or run as root."
	echo "  Typical: /var/log/apache2/error_log"
else
	DAY=$(echo "$DATE" | cut -d/ -f1 | sed 's/^0//')
	MON=$(echo "$DATE" | cut -d/ -f2)
	awk -v MON="$MON" -v DAY="$DAY" -v FROM="$FROM" -v TO="$TO" '
	{
		if (match($0, /\[[A-Z][a-z]{2} [A-Z][a-z]{2} [ 0-9][0-9] [0-9]{2}:[0-9]{2}:[0-9]{2}/)) {
			t = substr($0, RSTART+1, RLENGTH-1)
			split(t, p, " ")
			if (p[2] != MON) next
			if (p[3]+0 != DAY+0) next
			split(p[4], c, ":")
			hm = c[1] c[2]
			if (hm < FROM || hm > TO) next
			print
		}
	}' "$ERRLOG" > "$TMP/err.log"

	EC=$(wc -l < "$TMP/err.log")
	# Strip bot-scanner noise (client denied / .env & xmlrpc probing) — on a
	# shared cPanel box this is 90%+ of the file and drowns the real signal.
	grep -vE 'AH01630|AH01797|client denied by server configuration|AH01276|File does not exist' \
		"$TMP/err.log" > "$TMP/err_sig.log" 2>/dev/null || cp "$TMP/err.log" "$TMP/err_sig.log"
	ES=$(wc -l < "$TMP/err_sig.log")
	echo "Error-log lines in window : $EC"
	echo "  of which scanner noise  : $((EC - ES))  (client-denied probes, suppressed below)"
	echo "  signal lines            : $ES"
	if [ "$ES" -gt 0 ]; then
		echo
		echo "-- signature counts:"
		for sig in "Allowed memory size" "Timeout waiting for output" \
		           "End of script output before headers" "Premature end of script headers" \
		           "Maximum execution time" "MySQL server has gone away" \
		           "Fatal error" "Out of memory" "child process" "server reached MaxRequestWorkers"; do
			n=$(grep -cF "$sig" "$TMP/err_sig.log")
			[ "$n" -gt 0 ] && printf "  %6d  %s\n" "$n" "$sig"
		done
		echo
		echo "-- PHP memory-exhaustion by script (WHICH APP is eating RAM):"
		grep -F "Allowed memory size" "$TMP/err_sig.log" \
			| grep -oE 'in /[^ ]+\.php on line [0-9]+' | sort | uniq -c | sort -nr | head -$CAP_MD \
			| awk '{printf "  %6d  %s\n", $1, substr($0, index($0,$2))}'
		echo
		echo "   ^ on a shared cPanel box, check the /home/<user>/ prefix. Exhaustions"
		echo "     under someone else's docroot are another tenant's problem competing"
		echo "     for the same physical RAM as the ERP."
		echo
		echo "-- CGI timeouts — attributed by REFERER (the handler path is always"
		echo "   ea-phpNN and tells you nothing; the referer names the real screen):"
		grep -E "Timeout waiting for output|End of script output" "$TMP/err_sig.log" \
			| grep -oE 'referer: [^ ]+' | sed 's/^referer: //' \
			| sed -E 's/([?&](id|jobid|quoteid|purchaseid|dispatchid))=[0-9]+/\1=N/g' \
			| sort | uniq -c | sort -nr | head -$CAP_MD \
			| awk '{printf "  %6d  %s\n", $1, substr($0, index($0,$2))}'
		echo
		echo "   by client IP:"
		grep -E "Timeout waiting for output|End of script output" "$TMP/err_sig.log" \
			| grep -oE '\[client [0-9a-fA-F.:]+' | sed 's/\[client //' \
			| sort | uniq -c | sort -nr | head -$CAP_SM \
			| awk '{printf "  %6d  %s\n", $1, $2}'
		echo
		echo "-- signal errors by minute (scanner noise excluded):"
		awk '{ if (match($0, /[0-9]{2}:[0-9]{2}:[0-9]{2}/)) print substr($0, RSTART, 5) }' "$TMP/err_sig.log" \
			| sort | uniq -c | sort -k2 | awk '{printf "  %s  %5d\n", $2, $1}' | head -$CAP_LG
		echo
		echo "-- sample signal lines (first $CAP_SM):"
		head -$CAP_SM "$TMP/err_sig.log" | cut -c1-220 | sed 's/^/  /'
	fi
fi

###############################################################################
# Heuristic read
###############################################################################

sec "12. HEURISTIC READ"
PEAKMIN=$(sort -t $'\t' -k2,2nr "$TMP/reqmin.tsv" | head -1 | cut -f1)
PEAKCNT=$(sort -t $'\t' -k2,2nr "$TMP/reqmin.tsv" | head -1 | cut -f2)
PEAKIP=$(grep -E "^$PEAKMIN"$'\t' "$TMP/ipmin.tsv" | cut -f2)
echo "Peak minute      : ${PEAKMIN:0:2}:${PEAKMIN:2:2}  ($PEAKCNT requests, $PEAKIP distinct IPs)"
echo "5xx in window    : $FIVEXX"
echo "Bot requests     : $(awk -F'\t' '$9=="bot"' "$TMP/rec.tsv" | wc -l)"
echo "Heartbeat POSTs  : $(grep -c 'call=heartbeat' "$TMP/rec.tsv")  <- one per open tab per ~15s;"
echo "                    divide by window minutes for a concurrency estimate."
echo
echo "Next steps:"
echo "  * Zoom the spike:   ./forensic-web.sh -f ${PEAKMIN:0:2}:${PEAKMIN:2:2} -t ${PEAKMIN:0:2}:$(printf '%02d' $(( ${PEAKMIN:2:2} + 5 )) )"
echo "  * Host/DB side:     ./forensic-host.sh"
echo "  * Leave running:    nohup ./forensic-watch.sh > /dev/null 2>&1 &"

echo
hr
echo "Report written to: $OUT"
hr