INCIDENT 2026-08-29 18:21:11 trigger: php_rss=1630MB ================================================================== load: 2.21 2.00 1.59 2/666 584457 ncpu:10 mem_avail: 7121MB/13777MB (51.7%) swap_used:1485MB swapout:0/s blocked(D):0 running:2 iowait:0.0% steal:0.0% php: 12 procs / 1630MB heaviest user: madalice (9 procs, 1476MB) apache: 28 procs tcp_estab: 34 --- RSS by user (all processes) --- USER PROCS RSS_MB mysql 1 2814 madalice 9 1476 root 256 1090 nobody 27 507 netdata 7 198 mailman 20 190 otterfencing 1 114 cpanellogin 1 62 dovenull 6 56 wp-toolkit 3 43 mailnull 3 36 evolution 7 27 zabbix 1 22 dovecot 3 22 named 1 16 vinesit 1 14 lowrange 1 14 corpkicks 1 14 candyfabcom 1 14 alison 1 14 --- php workers by user --- USER PROCS RSS_MB madalice 9 1476 otterfencing 1 115 root 1 27 evolution 1 13 --- vm tuning / memory pressure --- vm.swappiness 30 vm.vfs_cache_pressure 100 vm.min_free_kbytes 67584 vm.overcommit_memory 0 vm.overcommit_ratio 50 Cached: 4726 MB Active(file): 4385 MB Inactive(file): 868 MB Dirty: 3 MB Writeback: 0 MB SReclaimable: 577 MB CommitLimit: 10984 MB Committed_AS: 8343 MB --- backup / dump / scan processes (page-cache evictors) --- PID USER ELAPSED RSS_KB COMMAND 489033 root 1-16:02:31 7112 /usr/bin/imunify-agent-proxy 497928 root 21:11 1332 /bin/sh -c bash -c 'sleep $((RANDOM % 1800))' ;imunify-antivirus imunify-patch subscriptions refresh > /dev/null 2>&1 --- top 60 by RSS --- PID PPID USER RSS %CPU STAT ELAPSED COMMAND 506671 1 mysql 2881936 9.0 Ssl 1-16:02:00 /usr/sbin/mariadbd 584414 510022 madalice 246268 32.7 S 00:04 php-fpm: pool madalice_com_au 508924 1 netdata 177848 1.9 Ssl 1-16:01:29 /usr/sbin/netdata -P /run/netdata/netdata.pid -D 83639 1451906 root 174448 0.0 S 02:23:37 spamd child 567420 510022 madalice 164260 8.9 S 03:22 php-fpm: pool madalice_com_au 567419 510022 madalice 162084 9.0 S 03:22 php-fpm: pool madalice_com_au 567421 510022 madalice 162000 8.6 S 03:22 php-fpm: pool madalice_com_au 567418 510022 madalice 161432 8.7 S 03:22 php-fpm: pool madalice_com_au 1502904 1451906 root 161056 0.0 S 15:44:49 spamd child 584401 510022 madalice 159712 41.3 S 00:06 php-fpm: pool madalice_com_au 584398 510022 madalice 159688 40.1 S 00:07 php-fpm: pool madalice_com_au 584406 510022 madalice 159632 45.8 S 00:05 php-fpm: pool madalice_com_au 584412 510022 madalice 137212 32.7 S 00:04 php-fpm: pool madalice_com_au 1451906 1 root 131424 0.0 Ss 15:58:04 /usr/local/cpanel/3rdparty/perl/542/bin/perl -T -w /usr/local/cpanel/3rdparty/bin/spamd --allowed-ips=127.0.0.1,::1 --max-children=5 -- 584427 583595 otterfencing 124572 93.0 R 00:01 /opt/cpanel/ea-php74/root/usr/bin/php-cgi 509266 1 root 112588 0.0 Ss 1-16:01:26 /usr/lib/systemd/systemd-journald 509236 509098 root 64472 0.0 Sl 1-16:01:26 /opt/saltstack/salt/bin/python3.10 /usr/bin/salt-minion MultiMinionProcessManager MinionProcessManager 463083 1353308 cpanellogin 64164 0.0 S 32:24 /usr/local/cpanel/base/show_template.stor cpanel_locale docroot /usr/local/cpanel/base default_login_theme cpanel server_locale en 1353308 1 root 42268 0.1 Ss 38-15:39:52 cpsrvd (SSL) - waiting for connections 509640 508992 root 38936 0.0 Sl 1-16:01:24 /usr/libexec/netdata/plugins.d/ebpf.plugin 1 1451116 1 root 33384 0.0 Ss 15:58:57 lfd - sleeping 510022 1 root 28324 0.0 Ss 1-16:01:22 php-fpm: master process (/opt/cpanel/ea-php83/root/etc/php-fpm.conf) 938582 1 wp-toolkit 25652 0.0 Ss 1-14:09:03 /usr/bin/sw-engine /usr/local/cpanel/3rdparty/wp-toolkit/bin/run-script background-tasks-executor.php 569172 569165 root 23532 0.0 S 01:31 /usr/sbin/exim -q 483898 1 zabbix 23396 0.2 Ssl 1-16:02:43 /usr/sbin/zabbix_agent2 -c /etc/zabbix/zabbix_agent2.conf 568588 506832 nobody 23020 0.0 S 02:04 /usr/sbin/httpd -k start 583656 506832 nobody 22684 0.1 S 00:54 /usr/sbin/httpd -k start 1469 1 root 22488 0.1 Ss 61-08:11:21 cpdavd - accepting connections on: 2079, 2080, 2091, 2077, 2078 583624 506832 nobody 22236 0.1 S 00:55 /usr/sbin/httpd -k start 568601 506832 nobody 22072 0.1 S 02:02 /usr/sbin/httpd -k start 567684 506832 nobody 21984 0.1 S 03:05 /usr/sbin/httpd -k start 583654 506832 nobody 21800 0.1 S 00:54 /usr/sbin/httpd -k start 583705 506832 nobody 21796 0.0 S 00:53 /usr/sbin/httpd -k start 566530 506832 nobody 21684 0.1 S 04:38 /usr/sbin/httpd -k start 583653 506832 nobody 21628 0.2 S 00:54 /usr/sbin/httpd -k start 1463 1 root 21388 0.0 Ss 61-08:11:21 tailwatchd 584048 506832 nobody 20868 0.1 S 00:36 /usr/sbin/httpd -k start 584046 506832 nobody 20632 0.0 S 00:36 /usr/sbin/httpd -k start 584394 506832 nobody 20600 0.5 S 00:08 /usr/sbin/httpd -k start 569179 569172 mailnull 20500 0.0 S 01:31 /usr/sbin/exim -q 567683 506832 nobody 20416 0.0 S 03:05 /usr/sbin/httpd -k start 583595 506832 nobody 20144 0.0 S 00:56 /usr/sbin/httpd -k start 584033 506832 nobody 20032 0.0 S 00:37 /usr/sbin/httpd -k start 584019 506832 nobody 19880 0.0 S 00:38 /usr/sbin/httpd -k start 584032 506832 nobody 19880 0.0 S 00:37 /usr/sbin/httpd -k start 584393 506832 nobody 19832 0.3 S 00:08 /usr/sbin/httpd -k start 583700 506832 nobody 19564 0.1 S 00:53 /usr/sbin/httpd -k start 584400 506832 nobody 19352 0.3 S 00:06 /usr/sbin/httpd -k start 584408 506832 nobody 19276 0.2 S 00:05 /usr/sbin/httpd -k start 569161 506832 nobody 19204 0.0 S 01:32 /usr/sbin/httpd -k start 584386 506832 nobody 17840 0.0 S 00:09 /usr/sbin/httpd -k start 508634 1 named 17400 0.0 Ssl 1-16:01:34 /usr/sbin/pdns_server --socket-dir=/run/pdns --guardian=no --daemon=no --disable-syslog --log-timestamp=no --write-pid=no 584409 506832 nobody 17272 0.0 S 00:05 /usr/sbin/httpd -k start 509671 508992 netdata 17056 2.1 Sl 1-16:01:24 /usr/libexec/netdata/plugins.d/apps.plugin 1 584418 506832 nobody 16752 0.0 S 00:03 /usr/sbin/httpd -k start 938585 1 wp-toolkit 16232 0.0 Ss 1-14:09:03 /usr/bin/sw-engine /usr/local/cpanel/3rdparty/wp-toolkit/bin/run-script scheduled-tasks-executor.php 2766563 1471 root 16228 0.0 S 09:43:23 cPhulkd - dbprocessor 1471 1 root 16036 0.0 Ss 61-08:11:21 cPhulkd - processor 584276 1471 root 15076 0.0 S 00:16 cPhulkd - processor - http socket 584421 508511 alison 14920 0.0 S 00:02 dovecot/imap --- processes in D state (uninterruptible IO) --- --- mariadb processlist (non-sleeping) --- +---------+------+------+---------+------+----------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ | ID | USER | DB | COMMAND | TIME | STATE | q | +---------+------+------+---------+------+----------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ | 4132697 | root | NULL | Query | 0 | Filling schema table | SELECT ID,USER,DB,COMMAND,TIME,STATE,LEFT(INFO,300) q FROM information_schema.PROCESSLIST WHERE COMMAND<>'Sleep' ORDER BY TIME DESC LIMIT 40 | +---------+------+------+---------+------+----------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ --- lock waits --- --- vhost traffic RIGHT NOW (last 3 min, deduped by inode) --- VHOST LOG REQ/3min otterfencing/otterfencing.com.au-ssl_log 211 madalice/madalice.com.au-ssl_log 116 aaa/aaadownunder.com.au-ssl_log 108 infinium/infinium.net.au-ssl_log 99 whiz/whizengineering.com.au 94 otterfencing/otterfencing.com.au 59 evolution/portal.evolutionerp.com.au-ssl_log 24 evolution/my.evolutionerp.com.au-ssl_log 5 (total vhosts taking traffic: 8) --- top client IPs across all vhosts (last 3 min) --- 211 20.48.251.3 107 20.104.49.130 97 162.254.39.130 94 20.48.250.41 77 20.151.129.194 57 4.205.62.107 24 203.54.180.238 4 203.123.106.85 4 118.127.40.55 2 84.246.212.174 2 57.129.136.36 2 50.87.179.84 2 50.6.169.131 2 42.194.192.163 2 40.77.167.20 --- top request paths across all vhosts (last 3 min) --- 86 GET /wp-login.php 24 GET /heartbeat.php 13 POST /wp-login.php 5 GET /store/hand-painted-nike-af1-afl-gws-goldcostsun/ 5 GET /ms-edit.php 5 GET / 4 GET /wp-content/plugins/hellopress/wp_filemanager.php 4 GET /thui.php 4 GET /this_is_a_new_hello_world.php 4 GET /f35.php 4 GET /8.php 3 POST /wp-cron.php 3 GET /vx.php 3 GET /ty.php 3 GET /erty.php 3 GET /chosen.php 3 GET /155.php 2 POST /wp-admin/admin-ajax.php 2 POST /functions.php 2 GET /xp.php --- busiest 3 vhosts: top paths + last 10 requests --- == otterfencing/otterfencing.com.au-ssl_log == -- top paths -- 2 GET /thui.php 2 GET /ms-edit.php 2 GET /koiy.php 2 GET /cloud.php 1 GET /zz.php 1 GET /zaza.php 1 GET /xza.php 1 GET /xve22.php 1 GET /xqq.php 1 GET /xp.php 1 GET /xmy.php 1 GET /xin1.php -- last 10 requests -- 20.48.251.3 - - [29/Aug/2026:18:20:36 +1000] "GET /linusadmin-phpinfo.php HTTP/1.1" 401 36 "-" "-" 20.48.251.3 - - [29/Aug/2026:18:20:36 +1000] "GET /app/Models/Hrm/Employee.php HTTP/1.1" 401 36 "-" "-" 20.48.251.3 - - [29/Aug/2026:18:20:37 +1000] "GET /agg.php HTTP/1.1" 401 36 "-" "-" 20.48.251.3 - - [29/Aug/2026:18:20:37 +1000] "GET /requirements.php HTTP/1.1" 401 36 "-" "-" 20.48.251.3 - - [29/Aug/2026:18:20:38 +1000] "GET /var/www/wallet/index.php HTTP/1.1" 401 36 "-" "-" 20.48.251.3 - - [29/Aug/2026:18:20:38 +1000] "GET /124.php HTTP/1.1" 401 36 "-" "-" 20.48.251.3 - - [29/Aug/2026:18:20:39 +1000] "GET /test1.php HTTP/1.1" 401 36 "-" "-" 20.48.251.3 - - [29/Aug/2026:18:20:39 +1000] "GET /bigdump.php HTTP/1.1" 401 36 "-" "-" 20.48.251.3 - - [29/Aug/2026:18:20:39 +1000] "GET /wdf.php HTTP/1.1" 401 36 "-" "-" 20.48.251.3 - - [29/Aug/2026:18:20:40 +1000] "GET /bb.php HTTP/1.1" 401 36 "-" "-" == madalice/madalice.com.au-ssl_log == -- top paths -- 5 GET /store/hand-painted-nike-af1-afl-gws-goldcostsun/ 5 GET / 2 POST /wp-admin/admin-ajax.php 1 POST / 1 GET /yj09.php 1 GET /xiugai.php 1 GET /xamp.php 1 GET /wp.php 1 GET /wp-json/wp/v2/users 1 GET /wp-json/wc-analytics/variations/v1/users 1 GET /wp-json/wc-analytics/variations/members 1 GET /wp-json/wc-analytics/taxes/v1/users -- last 10 requests -- 42.194.192.163 - - [29/Aug/2026:18:21:03 +1000] "GET / HTTP/1.1" 200 42739 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" 20.151.129.194 - - [29/Aug/2026:18:21:04 +1000] "GET /4PJcpMFsD8B.php HTTP/1.1" 400 44 "-" "-" 172.93.101.38 - - [29/Aug/2026:18:21:03 +1000] "GET /wp-json/wc-admin/experiments/users HTTP/1.1" 404 95 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/537.3 20.151.129.194 - - [29/Aug/2026:18:21:05 +1000] "GET /i99z7zzbwtpteujvv6s8hiCdefault.php HTTP/1.1" 400 44 "-" "-" 20.151.129.194 - - [29/Aug/2026:18:21:06 +1000] "GET /gwaih1gfzp5vuwr04Cdefault.php HTTP/1.1" 400 44 "-" "-" 20.151.129.194 - - [29/Aug/2026:18:21:06 +1000] "GET /1wvekeybd9it2di2vyipgr6Cdefault.php HTTP/1.1" 400 44 "-" "-" 20.151.129.194 - - [29/Aug/2026:18:21:07 +1000] "GET /media.php HTTP/1.1" 400 44 "-" "-" 162.214.184.71 - - [29/Aug/2026:18:21:06 +1000] "GET /wp-json/wc-admin/experiments/members HTTP/1.1" 404 95 "-" "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/151.0.0.0 Safari/53 20.151.129.194 - - [29/Aug/2026:18:21:08 +1000] "GET /inso.php HTTP/1.1" 400 44 "-" "-" 52.167.144.187 - - [29/Aug/2026:18:21:07 +1000] "GET /brand/vans-custom-sneakers/ HTTP/1.1" 200 35849 "-" "Mozilla/5.0 AppleWebKit/537.36 (KHTML, like Gecko; compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm) Chr == aaa/aaadownunder.com.au-ssl_log == -- top paths -- 1 GET /zoo2.php 1 GET /yawa.php 1 GET /xwx1.php 1 GET /xwpg.php 1 GET /xmini4.php 1 GET /xa.php 1 GET /x1da.php 1 GET /wsd.php 1 GET /ws85.php 1 GET /wpxml.php 1 GET /wp-the.php 1 GET /wp.php -- last 10 requests -- 20.104.49.130 - - [29/Aug/2026:18:20:36 +1000] "GET /wp-the.php HTTP/1.1" 404 5750 "-" "-" 20.104.49.130 - - [29/Aug/2026:18:20:37 +1000] "GET /155.php HTTP/1.1" 404 5750 "-" "-" 20.104.49.130 - - [29/Aug/2026:18:20:39 +1000] "GET /fs.php HTTP/1.1" 404 5750 "-" "-" 20.104.49.130 - - [29/Aug/2026:18:20:40 +1000] "GET /kgoc6awap3napxxxdCdefault.php HTTP/1.1" 404 5750 "-" "-" 20.104.49.130 - - [29/Aug/2026:18:20:48 +1000] "GET /ms-edit.php HTTP/1.1" 404 5750 "-" "-" 20.104.49.130 - - [29/Aug/2026:18:20:52 +1000] "GET /tool.php HTTP/1.1" 404 5750 "-" "-" 20.104.49.130 - - [29/Aug/2026:18:20:52 +1000] "GET /abc.php HTTP/1.1" 404 5750 "-" "-" 20.104.49.130 - - [29/Aug/2026:18:20:53 +1000] "GET /ioxi-o.php HTTP/1.1" 404 5750 "-" "-" 20.104.49.130 - - [29/Aug/2026:18:20:53 +1000] "GET /bthil.php HTTP/1.1" 404 5750 "-" "-" 20.104.49.130 - - [29/Aug/2026:18:20:53 +1000] "GET /xwx1.php HTTP/1.1" 404 5750 "-" "-" --- last 30 samples (run-up to this incident) --- 2026-08-29 18:18:40 L1=3.08 run=2 blk=0 avail=7153 (51.9%) swpO=0 conn=2 myRSS=2814 php=13 phpRSS=1685 ap=24 tcp=29 topuser=madalice 2026-08-29 18:18:50 L1=2.91 run=4 blk=0 avail=7170 (52.0%) swpO=0 conn=3 myRSS=2814 php=13 phpRSS=1669 ap=23 tcp=32 topuser=madalice 2026-08-29 18:18:51 L1=2.91 run=4 blk=0 avail=7159 (52.0%) swpO=0 conn=4 myRSS=2814 php=13 phpRSS=1686 ap=23 tcp=32 topuser=madalice 2026-08-29 18:19:01 L1=3.15 run=2 blk=0 avail=7005 (50.8%) swpO=0 conn=3 myRSS=2814 php=14 phpRSS=1870 ap=22 tcp=29 topuser=madalice 2026-08-29 18:19:01 L1=3.06 run=2 blk=0 avail=7001 (50.8%) swpO=0 conn=3 myRSS=2814 php=14 phpRSS=1874 ap=22 tcp=29 topuser=madalice 2026-08-29 18:19:11 L1=2.82 run=4 blk=0 avail=6775 (49.2%) swpO=0 conn=5 myRSS=2814 php=15 phpRSS=2098 ap=27 tcp=35 topuser=madalice 2026-08-29 18:19:11 L1=2.82 run=3 blk=0 avail=6765 (49.1%) swpO=0 conn=5 myRSS=2814 php=15 phpRSS=2111 ap=27 tcp=35 topuser=madalice 2026-08-29 18:19:21 L1=2.54 run=4 blk=0 avail=6951 (50.5%) swpO=2 conn=2 myRSS=2814 php=14 phpRSS=1832 ap=23 tcp=32 topuser=madalice 2026-08-29 18:19:21 L1=2.54 run=4 blk=0 avail=7028 (51.0%) swpO=2 conn=3 myRSS=2814 php=14 phpRSS=1837 ap=23 tcp=32 topuser=madalice 2026-08-29 18:19:31 L1=2.45 run=2 blk=0 avail=7341 (53.3%) swpO=0 conn=2 myRSS=2814 php=11 phpRSS=1453 ap=21 tcp=34 topuser=madalice 2026-08-29 18:19:31 L1=2.45 run=3 blk=0 avail=7307 (53.0%) swpO=0 conn=3 myRSS=2814 php=12 phpRSS=1516 ap=21 tcp=34 topuser=madalice 2026-08-29 18:19:41 L1=2.46 run=7 blk=0 avail=7443 (54.0%) swpO=0 conn=2 myRSS=2814 php=10 phpRSS=1315 ap=22 tcp=30 topuser=madalice 2026-08-29 18:19:41 L1=2.46 run=2 blk=0 avail=7580 (55.0%) swpO=0 conn=3 myRSS=2814 php=9 phpRSS=1149 ap=22 tcp=31 topuser=madalice 2026-08-29 18:19:51 L1=2.40 run=5 blk=0 avail=7226 (52.5%) swpO=0 conn=4 myRSS=2814 php=11 phpRSS=1571 ap=22 tcp=33 topuser=madalice 2026-08-29 18:19:51 L1=2.40 run=5 blk=0 avail=7233 (52.5%) swpO=0 conn=4 myRSS=2814 php=11 phpRSS=1561 ap=22 tcp=33 topuser=madalice 2026-08-29 18:20:01 L1=2.34 run=5 blk=0 avail=7507 (54.5%) swpO=0 conn=4 myRSS=2814 php=10 phpRSS=1255 ap=23 tcp=28 topuser=madalice 2026-08-29 18:20:01 L1=2.34 run=2 blk=0 avail=7537 (54.7%) swpO=0 conn=4 myRSS=2814 php=10 phpRSS=1212 ap=23 tcp=27 topuser=madalice 2026-08-29 18:20:11 L1=2.43 run=4 blk=0 avail=7357 (53.4%) swpO=0 conn=4 myRSS=2814 php=10 phpRSS=1288 ap=24 tcp=34 topuser=madalice 2026-08-29 18:20:11 L1=2.43 run=3 blk=0 avail=7420 (53.9%) swpO=0 conn=1 myRSS=2814 php=10 phpRSS=1165 ap=24 tcp=33 topuser=madalice 2026-08-29 18:20:21 L1=2.80 run=2 blk=0 avail=7420 (53.9%) swpO=0 conn=4 myRSS=2814 php=10 phpRSS=1215 ap=36 tcp=31 topuser=madalice 2026-08-29 18:20:21 L1=2.80 run=2 blk=0 avail=7395 (53.7%) swpO=0 conn=1 myRSS=2814 php=10 phpRSS=1238 ap=36 tcp=28 topuser=madalice 2026-08-29 18:20:31 L1=2.67 run=2 blk=0 avail=7546 (54.8%) swpO=0 conn=1 myRSS=2814 php=9 phpRSS=1139 ap=26 tcp=29 topuser=madalice 2026-08-29 18:20:31 L1=2.67 run=3 blk=0 avail=7543 (54.8%) swpO=0 conn=1 myRSS=2814 php=9 phpRSS=1142 ap=26 tcp=35 topuser=madalice 2026-08-29 18:20:41 L1=2.34 run=2 blk=0 avail=7535 (54.7%) swpO=0 conn=1 myRSS=2814 php=9 phpRSS=1128 ap=31 tcp=28 topuser=madalice 2026-08-29 18:20:41 L1=2.34 run=4 blk=0 avail=7596 (55.1%) swpO=3 conn=1 myRSS=2814 php=8 phpRSS=982 ap=30 tcp=28 topuser=madalice 2026-08-29 18:20:51 L1=2.05 run=2 blk=0 avail=7688 (55.8%) swpO=3 conn=1 myRSS=2814 php=9 phpRSS=1058 ap=23 tcp=22 topuser=madalice 2026-08-29 18:20:51 L1=2.05 run=2 blk=0 avail=7674 (55.7%) swpO=0 conn=1 myRSS=2814 php=9 phpRSS=1082 ap=23 tcp=23 topuser=madalice 2026-08-29 18:21:01 L1=1.89 run=2 blk=0 avail=7778 (56.5%) swpO=0 conn=1 myRSS=2814 php=8 phpRSS=955 ap=21 tcp=29 topuser=madalice 2026-08-29 18:21:02 L1=1.89 run=3 blk=0 avail=7842 (56.9%) swpO=0 conn=1 myRSS=2814 php=8 phpRSS=895 ap=21 tcp=31 topuser=madalice 2026-08-29 18:21:11 L1=2.21 run=2 blk=0 avail=7121 (51.7%) swpO=0 conn=3 myRSS=2814 php=12 phpRSS=1630 ap=28 tcp=34 topuser=madalice