#!/usr/bin/env bash
#
# sync-prod-logs.sh — pull production diagnostics onto the dev box.
#
# READ-ONLY with respect to production. Every transfer is an rsync PULL; the
# only commands run remotely are `find`, `du`, `test` and `ls`. Nothing on prod
# is written, rotated, truncated or restarted.
#
#   ./sync-prod-logs.sh                 sync everything readable, then digest
#   ./sync-prod-logs.sh --list          probe only: what exists, how big, readable?
#   ./sync-prod-logs.sh --dry-run       show the transfers, move nothing
#   ./sync-prod-logs.sh --only web-error,cron
#   ./sync-prod-logs.sh --since 30      widen the window on dated sources
#   ./sync-prod-logs.sh --full          no date window at all
#   ./sync-prod-logs.sh --domlogs       include per-vhost Apache logs (BIG)
#   ./sync-prod-logs.sh --quiet         cron mode: log to file, stdout only on error
#   ./sync-prod-logs.sh --install-cron  add an hourly crontab entry
#
# CREDENTIALS live outside the workspace, and therefore outside every git repo:
#
#   /config/secrets/prod-sync.env       (chmod 600)
#
# Run --list once with no config and it writes a commented template there.
#
set -Eeuo pipefail

CONF="${PROD_SYNC_CONF:-/config/secrets/prod-sync.env}"

# ---------------------------------------------------------------------------
# Defaults (override any of these in the config file)
# ---------------------------------------------------------------------------
PROD_HOST=""
PROD_USER="root"
PROD_PORT="22"
PROD_SSH_KEY=""
PROD_SSH_ALIAS=""          # set this to use a ~/.ssh/config Host block instead
PROD_PASSWORD=""           # last resort; key auth is strongly preferred
PROD_SUDO=0                # 1 = wrap remote rsync in sudo (for root-only logs)
PROD_WEBROOT="/home/evolution/my.evolutionerp.com.au"
PROD_HOMEDIR="/home/evolution"
PROD_FORENSIC_DIR="/var/log/erp-forensics"
# Relative to this script, not /config/workspace: the same tree is
# /var/www/shane.evolutionerp.com.au on the host, and a hard-coded container
# path run from the host writes to a stray /config/workspace there.
DEFAULT_DATA="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)/projects/production"
DATA="$DEFAULT_DATA"

DO_LIST=0 DRY_RUN=0 DO_DIGEST=1 QUIET=0 WANT_DOMLOGS=0 FULL=0
SINCE_DAYS=14
ONLY=""
CRON_SPEC=""
INSTALL_CRON=0

while [[ $# -gt 0 ]]; do
	case "$1" in
		--list)         DO_LIST=1; shift ;;
		--dry-run|-n)   DRY_RUN=1; shift ;;
		--only)         ONLY="$2"; shift 2 ;;
		--since)        SINCE_DAYS="$2"; shift 2 ;;
		--full)         FULL=1; shift ;;
		--domlogs)      WANT_DOMLOGS=1; shift ;;
		--no-digest)    DO_DIGEST=0; shift ;;
		--quiet|-q)     QUIET=1; shift ;;
		--install-cron) INSTALL_CRON=1; CRON_SPEC="${2:-17 * * * *}"
		                [[ "${2:-}" == --* || -z "${2:-}" ]] && CRON_SPEC="17 * * * *" || shift
		                shift ;;
		-h|--help)      sed -n '2,30p' "$0" | sed 's/^# \{0,1\}//'; exit 0 ;;
		*) echo "unknown option: $1" >&2; exit 2 ;;
	esac
done

SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")"
STAMP="$(date '+%Y-%m-%d_%H%M%S')"

# ---------------------------------------------------------------------------
# Config file. Written as a template on first run so the only manual step is
# filling in the host and dropping a key next to it.
# ---------------------------------------------------------------------------
if [[ ! -f "$CONF" ]]; then
	mkdir -p "$(dirname "$CONF")"
	chmod 700 "$(dirname "$CONF")" 2>/dev/null || true
	cat > "$CONF" <<'TEMPLATE'
# Production log-sync credentials.  KEEP THIS FILE OUT OF GIT.
# It lives outside /config/workspace precisely so it cannot be committed.
# chmod 600.

# --- connection ------------------------------------------------------------
PROD_HOST=""                 # e.g. my.evolutionerp.com.au (or the raw IP)
PROD_USER="root"             # root reaches the MariaDB/forensic/system logs
PROD_PORT="22"

# Key auth (preferred). Drop the private key beside this file, chmod 600.
PROD_SSH_KEY="/config/secrets/prod_id_ed25519"

# Alternatively, name a Host block from ~/.ssh/config and leave the three
# fields above empty:
PROD_SSH_ALIAS=""

# Password auth, only if there is genuinely no key. Needs sshpass.
PROD_PASSWORD=""

# The logs under /var/log are root-owned. If PROD_USER is not root but has
# passwordless sudo, set this to 1 and rsync runs as `sudo rsync` remotely.
PROD_SUDO=0

# --- remote paths (defaults suit the cPanel box) ---------------------------
PROD_WEBROOT="/home/evolution/my.evolutionerp.com.au"
PROD_HOMEDIR="/home/evolution"
PROD_FORENSIC_DIR="/var/log/erp-forensics"

# --- local ----------------------------------------------------------------
# The mirror always lands in ../projects/production beside the script. DATA is
# ignored here; export PROD_SYNC_DATA to put it somewhere else.
TEMPLATE
	chmod 600 "$CONF"
	echo "Wrote credentials template: $CONF"
	echo "Fill in PROD_HOST (and the key or password), then re-run."
	exit 3
fi

if [[ "$(stat -c '%a' "$CONF")" != "600" ]]; then
	echo "WARNING: $CONF is mode $(stat -c '%a' "$CONF"); tightening to 600." >&2
	chmod 600 "$CONF"
fi
# shellcheck disable=SC1090
source "$CONF"

# The local tree is not a credential, so the config does not get to move it.
# Older templates wrote DATA=/config/workspace/..., a path that exists only
# inside code-server; on the host it silently created a stray tree there.
if [[ -n "${PROD_SYNC_DATA:-}" ]]; then
	DATA="$PROD_SYNC_DATA"
elif [[ "$DATA" != "$DEFAULT_DATA" ]]; then
	echo "NOTE: ignoring DATA=$DATA from $CONF; using $DEFAULT_DATA" >&2
	DATA="$DEFAULT_DATA"
fi

# ---------------------------------------------------------------------------
# Local layout
# ---------------------------------------------------------------------------
LOGS="$DATA/logs"
HISTORY="$DATA/history"
STATE="$DATA/state"
DIGEST_DIR="$DATA/digest"
RUNLOG="$STATE/sync.log"
mkdir -p "$LOGS" "$HISTORY" "$STATE" "$DIGEST_DIR"

log() {
	printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" >> "$RUNLOG"
	[[ $QUIET -eq 1 ]] || printf '%s\n' "$*"
}
die() { log "ABORT: $*"; [[ $QUIET -eq 1 ]] && printf 'prod-log-sync ABORT: %s\n' "$*" >&2; exit 1; }
trap 'die "failed at line $LINENO"' ERR

if [[ $INSTALL_CRON -eq 1 ]]; then
	line="$CRON_SPEC $SELF --quiet"
	if crontab -l 2>/dev/null | grep -qF "sync-prod-logs.sh"; then
		echo "crontab entry already present:"
		crontab -l 2>/dev/null | grep -F "sync-prod-logs.sh"
	else
		# `crontab -l` exits 1 when the user has no crontab yet; under set -e that
		# killed the subshell and the ERR trap's ABORT text became the crontab.
		( crontab -l 2>/dev/null || true; echo "$line" ) | crontab -
		echo "installed: $line"
	fi
	exit 0
fi

# ---------------------------------------------------------------------------
# One run at a time. Two overlapping runs would race on the same rsync
# destinations and on the rotation guard below.
# ---------------------------------------------------------------------------
exec 9>"$STATE/.lock"
flock -n 9 || die "another sync is already running ($STATE/.lock)"

# ---------------------------------------------------------------------------
# SSH / rsync plumbing
# ---------------------------------------------------------------------------
SSH_OPTS=(-o ConnectTimeout=15 -o ServerAliveInterval=15 -o ServerAliveCountMax=4
          -o StrictHostKeyChecking=accept-new)

if [[ -n "$PROD_SSH_ALIAS" ]]; then
	TARGET="$PROD_SSH_ALIAS"
else
	[[ -n "$PROD_HOST" ]] || die "PROD_HOST is not set in $CONF"
	TARGET="${PROD_USER}@${PROD_HOST}"
	SSH_OPTS+=(-p "$PROD_PORT")
	if [[ -n "$PROD_SSH_KEY" ]]; then
		[[ -r "$PROD_SSH_KEY" ]] || die "PROD_SSH_KEY not readable: $PROD_SSH_KEY"
		# A key with loose permissions is silently ignored by ssh, which then
		# falls through to a password prompt and hangs a cron run forever.
		[[ "$(stat -c '%a' "$PROD_SSH_KEY")" =~ ^(600|400)$ ]] || chmod 600 "$PROD_SSH_KEY"
		SSH_OPTS+=(-i "$PROD_SSH_KEY" -o IdentitiesOnly=yes)
	fi
fi

SSH_WRAP=()
if [[ -n "$PROD_PASSWORD" ]]; then
	command -v sshpass >/dev/null || die "PROD_PASSWORD is set but sshpass is not installed (apt-get install sshpass)"
	export SSHPASS="$PROD_PASSWORD"
	SSH_WRAP=(sshpass -e)
	SSH_OPTS+=(-o PubkeyAuthentication=no -o PreferredAuthentications=password)
else
	# No password configured: never let ssh block on a prompt. A cron run that
	# hangs on "password:" holds the lock until someone notices.
	SSH_OPTS+=(-o BatchMode=yes)
fi

# -n: never read stdin. Inside a `while read` loop ssh would swallow the rest
# of the loop's input, silently ending the run after the first source.
ssh_run() { "${SSH_WRAP[@]}" ssh -n "${SSH_OPTS[@]}" "$TARGET" "$@"; }

# rsync needs the whole ssh invocation as one -e string.
RSH="${SSH_WRAP[*]:+${SSH_WRAP[*]} }ssh ${SSH_OPTS[*]}"

RSYNC_BASE=(rsync -rlptz --human-readable --stats --partial
            --timeout=120 -e "$RSH")
[[ "$PROD_SUDO" == "1" ]] && RSYNC_BASE+=(--rsync-path="sudo -n rsync")
[[ $DRY_RUN -eq 1 ]] && RSYNC_BASE+=(--dry-run)

# Remote command prefix for the probes, mirroring PROD_SUDO so a non-root
# account with sudo reports the same readability the transfers will see.
RCMD_PRE=""
[[ "$PROD_SUDO" == "1" ]] && RCMD_PRE="sudo -n "

log "=== prod log sync $STAMP — target $TARGET ==="
ssh_run true 2>/dev/null || die "cannot reach $TARGET over ssh (check $CONF; try: ssh ${SSH_OPTS[*]} $TARGET)"

# ---------------------------------------------------------------------------
# SOURCE TABLE
#
#   name | kind | rootonly | dated | remote glob-or-dir | local subdir | description
#
# kind=file  a single path or shell glob, pulled flat (catches error_log plus
#            its rotations error_log.1, error_log-20260901.gz, ...)
# kind=dir   a directory, pulled recursively
# dated=1    restricted to the last --since days (these dirs hold hundreds of
#            files under long retention; the whole history is rarely wanted)
# ---------------------------------------------------------------------------
MYSQL_LOGS="/var/lib/mysql/*.err /var/log/mysql*.log /var/log/mysqld.log /var/lib/mysql/*slow*.log /var/log/mysql/*.log"

SOURCES=(
  "web-error|file|0|0|$PROD_WEBROOT/error_log*|web|ERP vhost PHP/Apache error log"
  "cron|dir|0|0|$PROD_WEBROOT/cron/log|cron|cron.log, xero.log and friends"
  "deploy|file|0|0|$PROD_HOMEDIR/logs/*.log*|deploy|sync_production.sh deploy log"
  "tenant-error|file|0|0|$PROD_HOMEDIR/public_html/evolution_production/companies/./*/error_log|tenant|per-tenant plugin error logs"
  "mysql|file|1|0|$MYSQL_LOGS|mysql|MariaDB error log + slow query log"
  "forensics|dir|1|1|$PROD_FORENSIC_DIR|forensics|forensic-watch samples + incident dumps"
  "messages|file|1|0|/var/log/messages*|system|kernel ring buffer incl. OOM kills"
  "sar|dir|1|1|/var/log/sa|system/sa|sysstat history (sar -f)"
  "domlogs|dir|1|1|__DOMLOGS__|domlogs|per-vhost Apache access/error logs"
)

want() {
	local n="$1"
	[[ "$n" == domlogs && $WANT_DOMLOGS -eq 0 ]] && return 1
	[[ -z "$ONLY" ]] && return 0
	[[ ",$ONLY," == *",$n,"* ]]
}

# domlogs live in a different place on every cPanel/EA4 vintage.
resolve_domlogs() {
	ssh_run "sh -c '${RCMD_PRE}true; for d in /etc/apache2/logs/domlogs /var/log/apache2/domlogs /usr/local/apache/domlogs; do [ -d \"\$d\" ] && { printf \"%s\" \"\$d\"; break; }; done'" 2>/dev/null || true
}

# ---------------------------------------------------------------------------
# PROBE — for each candidate path: does it exist, is it readable, how big?
#
# Done before any transfer so an unreadable root-only source is reported as
# SKIPPED rather than failing the run. rsync's own error for an unreadable
# path is a bare exit 23, which says nothing about which source or why.
# ---------------------------------------------------------------------------
probe() {
	local glob="$1"
	ssh_run "sh -c '${RCMD_PRE}true; for p in $glob; do
		[ -e \"\$p\" ] || continue
		if [ -r \"\$p\" ]; then
			printf \"OK\t%s\t%s\n\" \"\$(du -sb \"\$p\" 2>/dev/null | cut -f1)\" \"\$p\"
		else
			printf \"DENIED\t0\t%s\n\" \"\$p\"
		fi
	done'" 2>/dev/null || true
}

human() { numfmt --to=iec --suffix=B "${1:-0}" 2>/dev/null || echo "${1}B"; }

SUMMARY="$STATE/sources.tsv"
: > "$SUMMARY.new"
total_bytes=0

for rec in "${SOURCES[@]}"; do
	IFS='|' read -r name kind rootonly dated remote localsub desc <<< "$rec"
	want "$name" || continue

	if [[ "$remote" == "__DOMLOGS__" ]]; then
		remote="$(resolve_domlogs)"
		[[ -n "$remote" ]] || { log "  SKIP  $name — no domlogs directory found on prod"; continue; }
	fi

	res="$(probe "$remote")"
	if [[ -z "$res" ]]; then
		log "  MISS  $name — nothing matched: $remote"
		printf '%s\tMISSING\t0\t%s\n' "$name" "$remote" >> "$SUMMARY.new"
		continue
	fi
	if ! grep -q '^OK' <<< "$res"; then
		log "  DENIED $name — exists but unreadable as $TARGET${rootonly:+ (root-only)}"
		[[ "$PROD_SUDO" == "1" ]] || log "         set PROD_USER=root, or PROD_SUDO=1 with passwordless sudo, in $CONF"
		printf '%s\tDENIED\t0\t%s\n' "$name" "$remote" >> "$SUMMARY.new"
		continue
	fi

	# A glob can match a mix of readable and unreadable paths. Reporting only
	# the readable ones would quietly understate the source: the interesting
	# tenant, or the one rotated log that matters, can be the unreadable one.
	denied=$(grep -c '^DENIED' <<< "$res" || true)
	if [[ "$denied" -gt 0 ]]; then
		log "  PARTIAL $name — $denied of $((denied + $(grep -c '^OK' <<< "$res"))) path(s) unreadable:"
		grep '^DENIED' <<< "$res" | cut -f3 | while IFS= read -r l; do log "          $l"; done
	fi

	bytes=$(awk -F'\t' '$1=="OK"{s+=$2} END{print s+0}' <<< "$res")
	count=$(grep -c '^OK' <<< "$res")
	total_bytes=$((total_bytes + bytes))
	log "  FOUND $name — $count path(s), $(human "$bytes")  [$desc]"
	printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' \
		"$name" "OK" "$bytes" "$kind" "$dated" "$remote" "$localsub" >> "$SUMMARY.new"
done

log "total on prod for the selected sources: $(human "$total_bytes")"

if [[ $DO_LIST -eq 1 ]]; then
	mv "$SUMMARY.new" "$SUMMARY"
	log "--list only; nothing transferred."
	exit 0
fi

# ---------------------------------------------------------------------------
# ROTATION GUARD
#
# A straight mirror LOSES history: when prod rotates error_log, our copy is
# truncated to match on the next run and everything before the rotation is
# gone. So before transferring a file, compare sizes — if the remote copy is
# now SMALLER than ours, it rotated, and our copy is archived first.
#
# Only a shrink triggers this. Appends (the normal case) must not, or every
# run would archive another full copy of a multi-hundred-MB log.
# ---------------------------------------------------------------------------
guard_rotation() {
	local remote_path="$1" local_path="$2" rsize lsize dest
	[[ -f "$local_path" ]] || return 0
	lsize=$(stat -c '%s' "$local_path")
	[[ "$lsize" -gt 0 ]] || return 0
	rsize=$(ssh_run "sh -c '${RCMD_PRE}du -sb \"$remote_path\" 2>/dev/null | cut -f1'" 2>/dev/null || echo "")
	[[ "$rsize" =~ ^[0-9]+$ ]] || return 0
	if [[ "$rsize" -lt "$lsize" ]]; then
		dest="$HISTORY/$STAMP/$(basename "$(dirname "$local_path")")"
		mkdir -p "$dest"
		log "  ROTATED $(basename "$local_path") on prod ($(human "$lsize") -> $(human "$rsize"))"
		log "          archiving our copy to ${dest#$DATA/}/"
		if [[ $DRY_RUN -eq 0 ]]; then
			gzip -c "$local_path" > "$dest/$(basename "$local_path").gz"
		fi
	fi
}

# ---------------------------------------------------------------------------
# TRANSFER
# ---------------------------------------------------------------------------
rc_total=0
# Read on fd 3, not stdin: rsync spawns ssh, which would consume stdin too.
while IFS=$'\t' read -r -u 3 name status bytes kind dated remote localsub; do
	[[ "$status" == "OK" ]] || continue
	dest="$LOGS/$localsub"
	mkdir -p "$dest"

	if [[ "$kind" == "file" ]]; then
		# A /./ in the source path means "keep everything after this point",
		# which needs rsync -R. Without it a per-tenant glob flattens.
		relative=()
		[[ "$remote" == */./* ]] && relative=(-R)

		# Archive any of our copies that prod has since rotated.
		for p in $(awk -F'\t' '$1=="OK"{print $3}' <<< "$(probe "$remote")"); do
			if [[ ${#relative[@]} -gt 0 ]]; then
				guard_rotation "$p" "$dest/${p#*/./}"
			else
				guard_rotation "$p" "$dest/$(basename "$p")"
			fi
		done
		log "  PULL  $name -> logs/$localsub/"
		# shellcheck disable=SC2086
		"${RSYNC_BASE[@]}" "${relative[@]}" "$TARGET":"$remote" "$dest/" >> "$RUNLOG" 2>&1 || {
			rc=$?; rc_total=$((rc_total+1)); log "  WARN  $name rsync exit $rc (see $RUNLOG)"; }
	else
		# Directory. Build the file list remotely so --since can apply; rsync
		# has no mtime filter of its own.
		listfile="$STATE/.files-$name"
		if [[ $FULL -eq 1 || "$dated" != "1" ]]; then
			findargs="-type f"
		else
			findargs="-type f -mtime -$SINCE_DAYS"
		fi
		ssh_run "sh -c '${RCMD_PRE}cd \"$remote\" 2>/dev/null && ${RCMD_PRE}find . $findargs -print'" \
			> "$listfile" 2>/dev/null || true
		n=$(grep -c . "$listfile" || true)
		if [[ "$n" -eq 0 ]]; then
			log "  EMPTY $name — no files in the last $SINCE_DAYS days"
			continue
		fi
		log "  PULL  $name -> logs/$localsub/ ($n files)"
		"${RSYNC_BASE[@]}" --files-from="$listfile" "$TARGET":"$remote/" "$dest/" >> "$RUNLOG" 2>&1 || {
			rc=$?; rc_total=$((rc_total+1)); log "  WARN  $name rsync exit $rc (see $RUNLOG)"; }
	fi
done 3< "$SUMMARY.new"

mv "$SUMMARY.new" "$SUMMARY"
date '+%Y-%m-%d %H:%M:%S %z' > "$STATE/last-sync.txt"

if [[ $DRY_RUN -eq 1 ]]; then
	log "dry run — nothing written."
	exit 0
fi

local_bytes=$(du -sb "$LOGS" 2>/dev/null | cut -f1)
log "local mirror now $(human "${local_bytes:-0}") at $LOGS"

# ---------------------------------------------------------------------------
# DIGEST
# ---------------------------------------------------------------------------
if [[ $DO_DIGEST -eq 1 ]]; then
	DIGEST="$(dirname "$SELF")/prod-log-digest.sh"
	if [[ -x "$DIGEST" ]]; then
		log "building digest"
		"$DIGEST" --data "$DATA" --since "$SINCE_DAYS" > "$DIGEST_DIR/digest-$STAMP.md" 2>>"$RUNLOG" || \
			log "WARN digest failed (see $RUNLOG)"
		ln -sfn "digest-$STAMP.md" "$DIGEST_DIR/latest.md"
		log "digest: ${DIGEST_DIR#$DATA/}/latest.md"
	else
		log "WARN prod-log-digest.sh not found next to this script — skipping digest"
	fi
fi

[[ $rc_total -eq 0 ]] || log "completed with $rc_total source(s) in error"
log "=== done ==="
exit 0