INCIDENT 2026-08-25 03:31:18 trigger: load=146.22/10cpu php_rss=1772MB ================================================================== load: 146.22 66.10 25.83 2/634 2362470 ncpu:10 mem_avail: 7345MB/11729MB (62.6%) swap_used:1351MB swapout:562/s blocked(D):3 running:5 iowait:10.6% steal:0.0% php: 9 procs / 1772MB heaviest user: madalice (9 procs, 1772MB) apache: 57 procs tcp_estab: 19 --- RSS by user (all processes) --- USER PROCS RSS_MB madalice 9 1772 nobody 56 796 root 255 629 netdata 8 225 mysql 1 200 dovenull 6 31 mailman 20 22 kurri 2 21 dovecot 3 16 vinesit 1 15 zabbix 1 14 wp-toolkit 6 14 candyfabcom 1 14 lowrange 2 12 cpanellogin 1 12 infinium 1 8 aaa 1 4 polkitd 1 3 cpanelconnecttrack 1 3 nscd 1 2 --- php workers by user --- USER PROCS RSS_MB madalice 9 1772 --- vm tuning / memory pressure --- vm.swappiness 30 vm.vfs_cache_pressure 100 vm.min_free_kbytes 67584 vm.overcommit_memory 0 vm.overcommit_ratio 50 Cached: 2170 MB Active(file): 306 MB Inactive(file): 681 MB Dirty: 2 MB Writeback: 0 MB SReclaimable: 147 MB CommitLimit: 9960 MB Committed_AS: 9313 MB --- backup / dump / scan processes (page-cache evictors) --- PID USER ELAPSED RSS_KB COMMAND 1913717 root 02:20:00 776 /usr/bin/imunify-agent-proxy 2361175 root 03:17 276 /bin/bash /etc/cron.daily/imunify-antivirus.cron 2361176 root 03:17 132 sed 1i\ /etc/cron.daily/imunify-antivirus.cron:\ 2361177 root 03:17 1228 /usr/libexec/report-command-error /opt/imunify360/venv/bin/python3 /opt/imunify360/venv/share/imunify360/scripts/delay_on_cron_call.py 2361188 root 03:17 1324 /opt/imunify360/venv/bin/python3 /opt/imunify360/venv/share/imunify360/scripts/delay_on_cron_call.py 2362087 root 01:06 1100 /bin/sh -c bash -c "sleep $((RANDOM % 60))" ; /opt/imunify360/venv/share/imunify360/scripts/check-detached.py > /dev/null 2>&1 || : --- top 60 by RSS --- PID PPID USER RSS %CPU STAT ELAPSED COMMAND 2361614 2361298 madalice 217272 5.1 S 02:59 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2361659 2326380 madalice 217256 5.7 S 02:59 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2361555 2361295 madalice 207936 5.6 S 03:00 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2362432 1 mysql 206260 0.0 Rsl 00:00 /usr/sbin/mariadbd 2361544 2361319 madalice 203964 5.2 S 03:00 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2361532 2361327 madalice 203300 6.2 S 03:00 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2361542 2361325 madalice 201168 5.3 S 03:00 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2361485 2361297 madalice 200632 2.4 S 03:00 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2361557 2361301 madalice 199280 6.1 S 03:00 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 1220 1 netdata 177336 2.5 Ssl 56-17:21:26 /usr/sbin/netdata -P /run/netdata/netdata.pid -D 2361718 2361684 madalice 164132 5.5 S 02:59 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2362098 2223 root 96968 7.0 Sl 00:12 /opt/saltstack/salt/bin/python3.10 /usr/bin/salt-minion MultiMinionProcessManager MinionProcessManager ProcessPayload(jid=2026082417300 2145741 1 root 92528 0.2 Ss 01:08:15 /usr/local/cpanel/3rdparty/perl/542/bin/perl -T -w /usr/local/cpanel/3rdparty/bin/spamd --allowed-ips=127.0.0.1,::1 --max-children=5 -- 2362344 2362098 root 91796 11.2 D 00:04 /usr/libexec/platform-python /usr/bin/dnf --quiet list --upgrades 2575382 1 root 86468 0.0 Ss 9-19:40:47 /usr/lib/systemd/systemd-journald 2160756 2145741 root 49780 0.0 S 01:05:53 spamd child 1353308 1 root 40132 0.0 Ss 34-00:49:56 cpsrvd (SSL) - waiting for connections 4922 2431 netdata 33148 0.1 Sl 56-17:21:21 /usr/libexec/netdata/plugins.d/go.d.plugin 1 2223 1201 root 31488 0.0 Sl 56-17:21:25 /opt/saltstack/salt/bin/python3.10 /usr/bin/salt-minion MultiMinionProcessManager MinionProcessManager 2160755 2145741 root 31360 0.0 S 01:05:53 spamd child 2361283 1276780 nobody 19980 0.0 S 03:06 /usr/sbin/httpd -k start 2362031 1276780 nobody 19864 3.3 S 02:33 /usr/sbin/httpd -k start 2361316 1276780 nobody 19800 4.2 S 03:03 /usr/sbin/httpd -k start 2362025 1276780 nobody 19664 2.9 S 02:37 /usr/sbin/httpd -k start 2345291 1276780 nobody 19652 0.0 S 06:30 /usr/sbin/httpd -k start 2361808 1276780 nobody 19564 0.0 S 02:58 /usr/sbin/httpd -k start 2362030 1276780 nobody 19392 2.9 S 02:34 /usr/sbin/httpd -k start 2342735 1276780 nobody 18292 1.0 S 10:23 /usr/sbin/httpd -k start 185104 2431 netdata 18188 2.1 Sl 12-06:57:44 /usr/libexec/netdata/plugins.d/apps.plugin 1 2362019 1276780 nobody 18044 2.4 S 02:41 /usr/sbin/httpd -k start 2327803 1276780 nobody 18004 0.0 S 12:37 /usr/sbin/httpd -k start 2342753 1276780 nobody 17776 0.0 S 10:19 /usr/sbin/httpd -k start 2345246 1276780 nobody 17608 0.0 S 06:35 /usr/sbin/httpd -k start 2359116 1276780 nobody 17548 0.0 S 06:00 /usr/sbin/httpd -k start 2359470 1276780 nobody 17228 2.3 S 05:25 /usr/sbin/httpd -k start 2361809 1276780 nobody 16624 4.0 S 02:58 /usr/sbin/httpd -k start 2361323 1276780 nobody 16464 0.0 S 03:03 /usr/sbin/httpd -k start 2361321 1276780 nobody 16460 0.0 S 03:03 /usr/sbin/httpd -k start 2361328 1276780 nobody 16456 0.0 S 03:03 /usr/sbin/httpd -k start 2361324 1276780 nobody 16440 0.0 S 03:03 /usr/sbin/httpd -k start 2361314 1276780 nobody 16436 0.0 S 03:03 /usr/sbin/httpd -k start 2361682 1276780 nobody 16412 0.0 S 02:59 /usr/sbin/httpd -k start 2361318 1276780 nobody 16408 0.0 S 03:03 /usr/sbin/httpd -k start 2361322 1276780 nobody 16408 0.0 S 03:03 /usr/sbin/httpd -k start 2361315 1276780 nobody 16404 0.0 S 03:03 /usr/sbin/httpd -k start 2361320 1276780 nobody 16400 0.0 S 03:03 /usr/sbin/httpd -k start 2361329 1276780 nobody 16400 0.0 S 03:03 /usr/sbin/httpd -k start 2361326 1276780 nobody 16396 0.0 S 03:03 /usr/sbin/httpd -k start 2361317 1276780 nobody 16376 0.0 S 03:03 /usr/sbin/httpd -k start 2361556 1276780 nobody 16244 0.0 S 03:00 /usr/sbin/httpd -k start 2361286 1276780 nobody 16072 0.0 S 03:05 /usr/sbin/httpd -k start 2360897 1276780 nobody 16064 0.0 S 03:36 /usr/sbin/httpd -k start 2361253 1276780 nobody 15904 0.0 S 03:07 /usr/sbin/httpd -k start 2361288 1276780 nobody 15888 0.0 S 03:05 /usr/sbin/httpd -k start 2361287 1276780 nobody 15856 0.0 S 03:05 /usr/sbin/httpd -k start 2362101 3828812 vinesit 15464 0.0 S 00:11 dovecot/imap 2361810 1276780 nobody 15456 3.6 S 02:58 /usr/sbin/httpd -k start 2326380 1276780 nobody 14492 0.0 S 14:43 /usr/sbin/httpd -k start 2362352 3828812 kurri 14464 0.0 S 00:03 dovecot/imap 2362334 3828812 candyfabcom 14436 0.0 S 00:04 dovecot/imap --- processes in D state (uninterruptible IO) --- 2362432 Dsl - /usr/sbin/mariadbd --- mariadb processlist (non-sleeping) --- +----+------+------+---------+------+----------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ | ID | USER | DB | COMMAND | TIME | STATE | q | +----+------+------+---------+------+----------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ | 3 | root | NULL | Query | 0 | Filling schema table | SELECT ID,USER,DB,COMMAND,TIME,STATE,LEFT(INFO,300) q FROM information_schema.PROCESSLIST WHERE COMMAND<>'Sleep' ORDER BY TIME DESC LIMIT 40 | +----+------+------+---------+------+----------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ --- lock waits --- --- vhost traffic RIGHT NOW (last 3 min, deduped by inode) --- VHOST LOG REQ/3min madalice/madalice.com.au-ssl_log 10 (total vhosts taking traffic: 1) --- top client IPs across all vhosts (last 3 min) --- 2 129.153.126.250 1 66.249.73.172 1 216.73.216.252 1 20.104.104.62 1 114.119.143.14 1 114.119.143.104 1 114.119.140.234 1 114.119.131.88 1 114.119.130.40 --- top request paths across all vhosts (last 3 min) --- 2 GET /tag/geelong 2 GET /store/custom-west-coast-eagles-shoes/ 2 GET /store/custom-venomous-shoes/ 1 GET /wp-content/plugins/admin.php 1 GET /store/sage-greens-kids-custom-painted-shoes/ 1 GET /store/custom-volley/ 1 GET /sitemap_index.xml --- busiest 3 vhosts: top paths + last 10 requests --- == madalice/madalice.com.au-ssl_log == -- top paths -- 2 GET /tag/geelong 2 GET /store/custom-west-coast-eagles-shoes/ 2 GET /store/custom-venomous-shoes/ 1 GET /wp-content/plugins/admin.php 1 GET /store/sage-greens-kids-custom-painted-shoes/ 1 GET /store/custom-volley/ 1 GET /sitemap_index.xml -- last 10 requests -- 66.249.73.172 - - [25/Aug/2026:03:29:58 +1000] "GET /store/sage-greens-kids-custom-painted-shoes/ HTTP/1.1" 500 833 "-" "Mozilla/5.0 (Linux; Android 6.0.1; Nexus 5X Build/MMB29P) AppleWebKit/537.36 (KHTML, like Gecko) Ch 114.119.130.40 - - [25/Aug/2026:03:31:07 +1000] "GET /store/custom-west-coast-eagles-shoes/?amp%3Bwcpr_rating=5&wcpr_page=4&wcpr_verified=1&image=1 HTTP/1.1" 500 833 "https://madalice.com.au/store/custom-west-coast-eagle 20.104.104.62 - - [25/Aug/2026:03:31:07 +1000] "GET /wp-content/plugins/admin.php HTTP/1.1" 500 833 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" 114.119.143.14 - - [25/Aug/2026:03:31:07 +1000] "GET /store/custom-volley/?amp%3Bverified=1&image=1&verified=1&rating=3 HTTP/1.1" 500 833 "https://madalice.com.au/store/custom-volley/?image=1&verified=1" "Mozilla/5.0 (Li 216.73.216.252 - - [25/Aug/2026:03:29:58 +1000] "GET /sitemap_index.xml HTTP/1.1" 500 833 "-" "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; ClaudeBot/1.0; +claudebot@anthropic.com)" 129.153.126.250 - - [25/Aug/2026:03:31:07 +1000] "GET /tag/geelong HTTP/1.1" 500 833 "-" "Mozilla/5.0 (X11; Linux aarch64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36" 114.119.131.88 - - [25/Aug/2026:03:31:08 +1000] "GET /store/custom-venomous-shoes/?wcpr_verified=1&wcpr_rating=3&rating=4 HTTP/1.1" 500 833 "https://madalice.com.au/store/custom-venomous-shoes/?wcpr_verified=1&wcpr_ratin 129.153.126.250 - - [25/Aug/2026:03:31:08 +1000] "GET /tag/geelong HTTP/1.1" 500 833 "-" "Mozilla/5.0 (X11; Linux aarch64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36" 114.119.140.234 - - [25/Aug/2026:03:30:03 +1000] "GET /store/custom-west-coast-eagles-shoes/?wcpr_rating=1&wcpr_verified=1&rating=3&wcpr_page=2 HTTP/1.1" 500 833 "https://madalice.com.au/store/custom-west-coast-eagles-sh 114.119.143.104 - - [25/Aug/2026:03:31:14 +1000] "GET /store/custom-venomous-shoes/?rating=5&wcpr_rating=5&wcpr_image=1 HTTP/1.1" 500 833 "https://madalice.com.au/store/custom-venomous-shoes/?rating=5&wcpr_rating=4&wcpr_ --- last 30 samples (run-up to this incident) --- 2026-08-25 03:26:11 L1=0.42 run=1 blk=0 avail=6488 (55.3%) swpO=0 conn=1 myRSS=2774 php=0 phpRSS=0 ap=18 tcp=23 topuser=- 2026-08-25 03:26:15 L1=0.47 run=2 blk=0 avail=6449 (55.0%) swpO=0 conn=2 myRSS=2774 php=1 phpRSS=61 ap=18 tcp=25 topuser=alison 2026-08-25 03:26:21 L1=0.43 run=2 blk=0 avail=6457 (55.1%) swpO=1 conn=1 myRSS=2774 php=1 phpRSS=59 ap=18 tcp=27 topuser=madalice 2026-08-25 03:26:25 L1=0.47 run=1 blk=0 avail=6480 (55.2%) swpO=1 conn=2 myRSS=2774 php=0 phpRSS=0 ap=18 tcp=26 topuser=- 2026-08-25 03:26:31 L1=0.44 run=2 blk=0 avail=6482 (55.3%) swpO=0 conn=1 myRSS=2774 php=0 phpRSS=0 ap=18 tcp=25 topuser=- 2026-08-25 03:26:35 L1=0.40 run=1 blk=0 avail=6485 (55.3%) swpO=0 conn=2 myRSS=2774 php=0 phpRSS=0 ap=18 tcp=24 topuser=- 2026-08-25 03:26:41 L1=0.37 run=1 blk=0 avail=6492 (55.4%) swpO=0 conn=1 myRSS=2774 php=0 phpRSS=0 ap=17 tcp=24 topuser=- 2026-08-25 03:26:45 L1=0.42 run=1 blk=0 avail=6487 (55.3%) swpO=0 conn=2 myRSS=2774 php=0 phpRSS=0 ap=17 tcp=24 topuser=- 2026-08-25 03:26:51 L1=0.47 run=1 blk=0 avail=6501 (55.4%) swpO=0 conn=1 myRSS=2774 php=0 phpRSS=0 ap=16 tcp=25 topuser=- 2026-08-25 03:26:55 L1=0.43 run=1 blk=0 avail=6504 (55.5%) swpO=0 conn=1 myRSS=2774 php=0 phpRSS=0 ap=16 tcp=22 topuser=- 2026-08-25 03:27:01 L1=0.39 run=1 blk=0 avail=6510 (55.5%) swpO=0 conn=1 myRSS=2774 php=0 phpRSS=0 ap=15 tcp=24 topuser=- 2026-08-25 03:27:05 L1=0.36 run=1 blk=0 avail=6505 (55.5%) swpO=0 conn=1 myRSS=2774 php=0 phpRSS=0 ap=15 tcp=24 topuser=- 2026-08-25 03:27:11 L1=0.57 run=1 blk=0 avail=6506 (55.5%) swpO=0 conn=1 myRSS=2774 php=0 phpRSS=0 ap=15 tcp=21 topuser=- 2026-08-25 03:27:16 L1=0.69 run=3 blk=0 avail=6371 (54.3%) swpO=0 conn=1 myRSS=2774 php=1 phpRSS=168 ap=15 tcp=22 topuser=madalice 2026-08-25 03:27:21 L1=0.79 run=1 blk=0 avail=6509 (55.5%) swpO=0 conn=1 myRSS=2774 php=0 phpRSS=0 ap=15 tcp=25 topuser=- 2026-08-25 03:27:26 L1=0.89 run=1 blk=0 avail=6508 (55.5%) swpO=0 conn=1 myRSS=2774 php=0 phpRSS=0 ap=15 tcp=23 topuser=- 2026-08-25 03:27:31 L1=0.90 run=1 blk=0 avail=6516 (55.6%) swpO=2 conn=1 myRSS=2774 php=0 phpRSS=0 ap=15 tcp=23 topuser=- 2026-08-25 03:27:36 L1=0.91 run=1 blk=0 avail=6507 (55.5%) swpO=2 conn=1 myRSS=2774 php=0 phpRSS=0 ap=15 tcp=23 topuser=- 2026-08-25 03:27:41 L1=0.91 run=5 blk=0 avail=6398 (54.5%) swpO=2 conn=1 myRSS=2774 php=3 phpRSS=200 ap=15 tcp=28 topuser=tubesupplies 2026-08-25 03:27:46 L1=1.08 run=2 blk=0 avail=6374 (54.3%) swpO=2 conn=1 myRSS=2774 php=1 phpRSS=156 ap=16 tcp=24 topuser=customc 2026-08-25 03:27:51 L1=1.08 run=1 blk=0 avail=6505 (55.5%) swpO=0 conn=1 myRSS=2774 php=0 phpRSS=0 ap=16 tcp=19 topuser=- 2026-08-25 03:27:56 L1=1.07 run=2 blk=0 avail=6502 (55.4%) swpO=0 conn=1 myRSS=2774 php=0 phpRSS=0 ap=14 tcp=19 topuser=- 2026-08-25 03:28:01 L1=1.06 run=1 blk=0 avail=6503 (55.4%) swpO=0 conn=1 myRSS=2774 php=0 phpRSS=0 ap=14 tcp=22 topuser=- 2026-08-25 03:28:06 L1=1.06 run=1 blk=0 avail=6501 (55.4%) swpO=0 conn=1 myRSS=2774 php=0 phpRSS=0 ap=14 tcp=23 topuser=- 2026-08-25 03:28:11 L1=1.05 run=10 blk=0 avail=5795 (49.4%) swpO=0 conn=9 myRSS=2774 php=8 phpRSS=952 ap=15 tcp=83 topuser=madalice 2026-08-25 03:28:16 L1=2.33 run=21 blk=0 avail=4485 (38.2%) swpO=0 conn=17 myRSS=2774 php=16 phpRSS=2407 ap=45 tcp=79 topuser=madalice 2026-08-25 03:28:22 L1=4.55 run=38 blk=0 avail=2745 (23.4%) swpO=2399 conn=37 myRSS=2704 php=36 phpRSS=4712 ap=50 tcp=58 topuser=madalice 2026-08-25 03:29:01 L1=63.15 run=191 blk=1 avail=78 (0.7%) swpO=2987 conn=- myRSS=0 php=36 phpRSS=6634 ap=62 tcp=41 topuser=madalice 2026-08-25 03:28:58 L1=54.28 run=186 blk=3 avail=79 (0.7%) swpO=6348 conn=- myRSS=0 php=25 phpRSS=4827 ap=62 tcp=32 topuser=madalice 2026-08-25 03:31:18 L1=146.22 run=5 blk=3 avail=7345 (62.6%) swpO=562 conn=- myRSS=193 php=9 phpRSS=1772 ap=57 tcp=19 topuser=madalice