INCIDENT 2026-08-15 07:50:51 trigger: load=260.46/10cpu php_procs=40 php_rss=7151MB mysql_conn=42 ================================================================== load: 260.46 203.72 99.98 35/813 2576202 ncpu:10 mem_avail: 2932MB/11729MB (25.0%) swap_used:2044MB swapout:166/s blocked(D):2 running:46 iowait:9.7% steal:0.0% php: 40 procs / 7151MB heaviest user: madalice (39 procs, 7084MB) apache: 135 procs tcp_estab: 19 --- RSS by user (all processes) --- USER PROCS RSS_MB madalice 39 7305 nobody 134 1392 mysql 1 475 root 256 352 netdata 8 189 aaa 1 75 mailman 20 25 candyfabcom 2 21 wp-toolkit 6 17 dovenull 6 13 zabbix 1 10 corpkicks 1 10 dovecot 3 7 cpanelphpmyadmin 1 5 evolution 4 3 dbus 1 3 cpanelconnecttrack 1 3 polkitd 1 2 nscd 1 1 named 1 1 --- php workers by user --- USER PROCS RSS_MB madalice 39 7347 aaa 1 75 --- vm tuning / memory pressure --- vm.swappiness 30 vm.vfs_cache_pressure 100 vm.min_free_kbytes 67584 vm.overcommit_memory 0 vm.overcommit_ratio 50 Cached: 5128 MB Active(file): 464 MB Inactive(file): 257 MB Dirty: 0 MB Writeback: 0 MB SReclaimable: 143 MB CommitLimit: 9960 MB Committed_AS: 16317 MB --- backup / dump / scan processes (page-cache evictors) --- PID USER ELAPSED RSS_KB COMMAND 2342983 root 06:39:36 0 /usr/bin/imunify-agent-proxy 2575381 root 00:15 1212 /bin/sh -c bash -c "sleep $((RANDOM % 60))" ; /opt/imunify360/venv/share/imunify360/scripts/check-detached.py > /dev/null 2>&1 || : --- top 60 by RSS --- PID PPID USER RSS %CPU STAT ELAPSED COMMAND 2575700 1 mysql 487068 12.5 Ssl 00:12 /usr/sbin/mariadbd 2575214 2574577 madalice 220644 12.1 D 00:20 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575218 2574027 madalice 220636 12.7 R 00:20 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575389 2574461 madalice 220612 16.0 R 00:15 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575334 2574028 madalice 220608 13.5 R 00:16 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575289 2574460 madalice 220592 14.1 R 00:18 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575291 2574017 madalice 220392 14.7 R 00:18 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575281 2573020 madalice 220376 13.8 R 00:18 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575209 2574225 madalice 220328 12.3 D 00:20 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575515 2574069 madalice 220300 16.9 R 00:13 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575257 2574578 madalice 220224 12.7 R 00:19 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575509 2575340 madalice 220208 17.0 R 00:13 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575325 2574026 madalice 219816 14.8 R 00:16 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575299 2574459 madalice 219744 15.1 R 00:18 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575333 2574067 madalice 219320 15.3 R 00:16 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575520 2575411 madalice 219316 16.6 S 00:13 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575212 2574296 madalice 218812 12.7 R 00:20 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575123 2573694 madalice 218232 5.8 R 00:53 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575107 2573954 madalice 216588 4.6 R 01:11 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575210 2574298 madalice 210160 12.8 R 00:20 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575119 2574064 madalice 207600 5.9 R 01:00 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575511 2575407 madalice 206628 16.7 R 00:13 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575930 2575408 madalice 205100 22.4 R 00:10 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575261 2571825 madalice 203980 12.4 R 00:18 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575518 2574902 madalice 199344 18.3 R 00:13 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575297 2574294 madalice 195992 12.3 R 00:18 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575118 2571821 madalice 193008 4.5 R 01:04 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575199 2574601 madalice 190908 11.3 R 00:20 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575267 2574066 madalice 190660 12.8 R 00:18 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575305 2573992 madalice 186116 11.2 R 00:17 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575219 2573990 madalice 185980 11.7 R 00:20 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575216 2571805 madalice 180496 9.8 R 00:20 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575295 2574293 madalice 178436 10.0 R 00:18 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575198 2573993 madalice 178116 10.3 R 00:20 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575274 2574458 madalice 174088 11.7 R 00:18 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575102 2573991 madalice 162156 3.6 R 01:24 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575284 2571044 madalice 155748 9.2 R 00:18 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575535 2575455 madalice 153560 12.6 R 00:13 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 1220 1 netdata 151800 2.5 Ssl 46-21:40:54 /usr/sbin/netdata -P /run/netdata/netdata.pid -D 2574390 2574017 madalice 135336 3.5 D 07:28 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2575147 2573953 madalice 133144 8.2 R 00:20 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2390233 2382613 root 108168 0.0 S 05:15:14 spamd child 2576081 2575541 aaa 78280 12.0 D 00:05 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2576203 2575396 root 25744 0.0 R 00:00 /usr/local/cpanel/scripts/restartsrv_cpanel_php_fpm --check --notconfigured-ok 4922 2431 netdata 23260 0.1 Sl 46-21:40:49 /usr/libexec/netdata/plugins.d/go.d.plugin 1 2382613 1 root 23236 0.1 Ss 05:27:40 /usr/local/cpanel/3rdparty/perl/542/bin/perl -T -w /usr/local/cpanel/3rdparty/bin/spamd --allowed-ips=127.0.0.1,::1 --max-children=5 -- 2575396 1463 root 21596 1.0 S 00:15 tailwatchd - chkservd - cpanel_php_fpm check 2576229 1469 root 20776 0.0 S 00:00 cpdavd - accepting connections on: 2079, 2080, 2091, 2077, 2078 2575287 1276780 nobody 18516 0.4 S 00:18 /usr/sbin/httpd -k start 2575413 1276780 nobody 18396 0.4 S 00:15 /usr/sbin/httpd -k start 2575055 1276780 nobody 18268 1.5 S 05:40 /usr/sbin/httpd -k start 2575313 1276780 nobody 18256 0.1 S 00:17 /usr/sbin/httpd -k start 2574887 1276780 nobody 17936 1.8 S 07:07 /usr/sbin/httpd -k start 2575342 1276780 nobody 17896 0.3 S 00:16 /usr/sbin/httpd -k start 2574904 1276780 nobody 17636 2.1 S 07:05 /usr/sbin/httpd -k start 2574865 1276780 nobody 17464 0.4 S 07:09 /usr/sbin/httpd -k start 2574798 1276780 nobody 17448 0.3 S 07:14 /usr/sbin/httpd -k start 2574831 1276780 nobody 17348 0.3 S 07:13 /usr/sbin/httpd -k start 2574830 1276780 nobody 17316 0.3 S 07:13 /usr/sbin/httpd -k start 2574829 1276780 nobody 16880 0.0 S 07:13 /usr/sbin/httpd -k start --- processes in D state (uninterruptible IO) --- 2576125 D - php-fpm: pool cpanelphpmyadmin --- mariadb processlist (non-sleeping) --- +----+------+------+---------+------+----------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ | ID | USER | DB | COMMAND | TIME | STATE | q | +----+------+------+---------+------+----------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ | 77 | root | NULL | Query | 0 | Filling schema table | SELECT ID,USER,DB,COMMAND,TIME,STATE,LEFT(INFO,300) q FROM information_schema.PROCESSLIST WHERE COMMAND<>'Sleep' ORDER BY TIME DESC LIMIT 40 | +----+------+------+---------+------+----------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ --- lock waits --- --- vhost traffic RIGHT NOW (last 3 min, deduped by inode) --- VHOST LOG REQ/3min madalice/madalice.com.au-ssl_log 10 evolution/portal.evolutionerp.com.au-ssl_log 5 (total vhosts taking traffic: 2) --- top client IPs across all vhosts (last 3 min) --- 5 203.54.180.238 1 84.104.44.110 1 5.199.239.199 1 45.163.75.189 1 37.243.63.243 1 201.1.123.111 1 186.40.9.237 1 170.84.181.28 1 160.238.191.148 1 105.113.108.206 1 102.101.112.7 --- top request paths across all vhosts (last 3 min) --- 5 GET /heartbeat.php 1 GET /tag/packs/ 1 GET /store/sprinkles-kids-custom-shoes/ 1 GET /store/nike-gold-eyelets/ 1 GET /store/dont-be-a-goose-custom-nike-air-force-1/ 1 GET /store/colour-choice-cartoon-drippy-custom-shoes/ 1 GET /store/choc-donut/ 1 GET /product-tag/sketchy/ 1 GET /product-tag/shoes/ 1 GET /product-tag/floral/ 1 GET /product-tag/adidas/ --- busiest 3 vhosts: top paths + last 10 requests --- == madalice/madalice.com.au-ssl_log == -- top paths -- 1 GET /tag/packs/ 1 GET /store/sprinkles-kids-custom-shoes/ 1 GET /store/nike-gold-eyelets/ 1 GET /store/dont-be-a-goose-custom-nike-air-force-1/ 1 GET /store/colour-choice-cartoon-drippy-custom-shoes/ 1 GET /store/choc-donut/ 1 GET /product-tag/sketchy/ 1 GET /product-tag/shoes/ 1 GET /product-tag/floral/ 1 GET /product-tag/adidas/ -- last 10 requests -- 160.238.191.148 - - [15/Aug/2026:07:50:38 +1000] "GET /product-tag/floral/ HTTP/1.1" 500 833 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36" 84.104.44.110 - - [15/Aug/2026:07:50:38 +1000] "GET /product-tag/adidas/ HTTP/1.1" 500 833 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36" 102.101.112.7 - - [15/Aug/2026:07:50:38 +1000] "GET /product-tag/shoes/ HTTP/1.1" 500 833 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36" 5.199.239.199 - - [15/Aug/2026:07:50:38 +1000] "GET /store/sprinkles-kids-custom-shoes/ HTTP/1.1" 500 833 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safar 201.1.123.111 - - [15/Aug/2026:07:50:38 +1000] "GET /tag/packs/ HTTP/1.1" 500 833 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36" 186.40.9.237 - - [15/Aug/2026:07:50:38 +1000] "GET /store/colour-choice-cartoon-drippy-custom-shoes/ HTTP/1.1" 500 833 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/14 105.113.108.206 - - [15/Aug/2026:07:50:38 +1000] "GET /store/dont-be-a-goose-custom-nike-air-force-1/ HTTP/1.1" 500 833 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/1 37.243.63.243 - - [15/Aug/2026:07:50:38 +1000] "GET /store/choc-donut/ HTTP/1.1" 500 833 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36" 45.163.75.189 - - [15/Aug/2026:07:50:38 +1000] "GET /product-tag/sketchy/ HTTP/1.1" 500 833 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36" 170.84.181.28 - - [15/Aug/2026:07:50:38 +1000] "GET /store/nike-gold-eyelets/ HTTP/1.1" 500 833 "-" "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36" == evolution/portal.evolutionerp.com.au-ssl_log == -- top paths -- 5 GET /heartbeat.php -- last 10 requests -- 203.54.180.238 - - [15/Aug/2026:07:50:38 +1000] "GET /heartbeat.php?sentAt=1786744233&bytes=106,127,141,169&_=1786683795473 HTTP/1.1" 500 20 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6 203.54.180.238 - - [15/Aug/2026:07:50:38 +1000] "GET /heartbeat.php?sentAt=1786744216&bytes=106,127,141,152&_=1786712385439 HTTP/1.1" 500 20 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6 203.54.180.238 - - [15/Aug/2026:07:50:38 +1000] "GET /heartbeat.php?sentAt=1786744163&bytes=106,127,141,99&_=1786683795466 HTTP/1.1" 500 20 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6" 203.54.180.238 - - [15/Aug/2026:07:50:41 +1000] "GET /heartbeat.php?sentAt=1786744226&bytes=106,127,141,162&_=1786712385440 HTTP/1.1" 500 20 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6 203.54.180.238 - - [15/Aug/2026:07:50:43 +1000] "GET /heartbeat.php?sentAt=1786744243&bytes=106,127,141,179&_=1786683795474 HTTP/1.1" 200 10 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6 --- last 30 samples (run-up to this incident) --- 2026-08-15 07:41:40 L1=2.07 run=1 blk=0 avail=9424 (80.3%) swpO=0 conn=2 myRSS=546 php=0 phpRSS=0 ap=22 tcp=22 topuser=- 2026-08-15 07:41:46 L1=1.91 run=1 blk=0 avail=9419 (80.3%) swpO=0 conn=3 myRSS=546 php=1 phpRSS=40 ap=21 tcp=21 topuser=evolution 2026-08-15 07:41:50 L1=1.83 run=1 blk=0 avail=9439 (80.5%) swpO=0 conn=2 myRSS=546 php=0 phpRSS=0 ap=20 tcp=20 topuser=- 2026-08-15 07:41:56 L1=1.69 run=2 blk=0 avail=9428 (80.4%) swpO=0 conn=3 myRSS=546 php=1 phpRSS=40 ap=19 tcp=21 topuser=evolution 2026-08-15 07:42:00 L1=1.63 run=1 blk=0 avail=9438 (80.5%) swpO=0 conn=2 myRSS=546 php=0 phpRSS=0 ap=19 tcp=21 topuser=- 2026-08-15 07:42:06 L1=1.58 run=3 blk=0 avail=9421 (80.3%) swpO=0 conn=3 myRSS=546 php=1 phpRSS=40 ap=19 tcp=22 topuser=evolution 2026-08-15 07:42:10 L1=1.45 run=1 blk=0 avail=9472 (80.8%) swpO=0 conn=2 myRSS=546 php=0 phpRSS=0 ap=19 tcp=18 topuser=- 2026-08-15 07:42:16 L1=1.34 run=2 blk=0 avail=9457 (80.6%) swpO=0 conn=2 myRSS=546 php=1 phpRSS=40 ap=19 tcp=20 topuser=evolution 2026-08-15 07:42:20 L1=1.23 run=1 blk=0 avail=9483 (80.9%) swpO=0 conn=1 myRSS=546 php=0 phpRSS=0 ap=17 tcp=19 topuser=- 2026-08-15 07:42:26 L1=1.29 run=3 blk=0 avail=9291 (79.2%) swpO=0 conn=2 myRSS=546 php=3 phpRSS=285 ap=17 tcp=22 topuser=olympic 2026-08-15 07:42:30 L1=1.27 run=2 blk=0 avail=9490 (80.9%) swpO=0 conn=1 myRSS=546 php=0 phpRSS=0 ap=17 tcp=21 topuser=- 2026-08-15 07:42:36 L1=1.17 run=2 blk=0 avail=9429 (80.4%) swpO=0 conn=2 myRSS=546 php=2 phpRSS=117 ap=17 tcp=21 topuser=olympic 2026-08-15 07:42:40 L1=1.23 run=2 blk=0 avail=9484 (80.9%) swpO=0 conn=1 myRSS=546 php=0 phpRSS=0 ap=18 tcp=19 topuser=- 2026-08-15 07:42:46 L1=1.13 run=1 blk=0 avail=9460 (80.7%) swpO=0 conn=2 myRSS=546 php=2 phpRSS=69 ap=21 tcp=20 topuser=evolution 2026-08-15 07:42:50 L1=1.04 run=1 blk=0 avail=9467 (80.7%) swpO=0 conn=1 myRSS=546 php=0 phpRSS=0 ap=20 tcp=24 topuser=- 2026-08-15 07:42:56 L1=1.04 run=2 blk=0 avail=9420 (80.3%) swpO=0 conn=2 myRSS=547 php=1 phpRSS=40 ap=20 tcp=24 topuser=evolution 2026-08-15 07:43:00 L1=0.96 run=1 blk=0 avail=9430 (80.4%) swpO=0 conn=1 myRSS=547 php=0 phpRSS=0 ap=20 tcp=24 topuser=- 2026-08-15 07:43:06 L1=0.88 run=1 blk=0 avail=9424 (80.3%) swpO=0 conn=2 myRSS=547 php=1 phpRSS=40 ap=19 tcp=24 topuser=evolution 2026-08-15 07:43:10 L1=1.37 run=10 blk=0 avail=8623 (73.5%) swpO=0 conn=1 myRSS=547 php=7 phpRSS=988 ap=26 tcp=41 topuser=madalice 2026-08-15 07:43:16 L1=1.98 run=12 blk=0 avail=8535 (72.8%) swpO=0 conn=11 myRSS=549 php=11 phpRSS=1078 ap=41 tcp=85 topuser=madalice 2026-08-15 07:43:21 L1=3.10 run=30 blk=0 avail=7095 (60.5%) swpO=0 conn=22 myRSS=549 php=21 phpRSS=2763 ap=52 tcp=117 topuser=madalice 2026-08-15 07:43:27 L1=5.50 run=54 blk=0 avail=4744 (40.4%) swpO=0 conn=40 myRSS=551 php=44 phpRSS=5517 ap=71 tcp=155 topuser=madalice 2026-08-15 07:43:34 L1=10.34 run=68 blk=0 avail=2706 (23.1%) swpO=38 conn=58 myRSS=558 php=59 phpRSS=7958 ap=87 tcp=186 topuser=madalice 2026-08-15 07:43:37 L1=15.36 run=79 blk=0 avail=1634 (13.9%) swpO=505 conn=67 myRSS=561 php=74 phpRSS=9131 ap=89 tcp=172 topuser=madalice 2026-08-15 07:43:44 L1=27.28 run=69 blk=24 avail=383 (3.3%) swpO=15506 conn=79 myRSS=563 php=85 phpRSS=11291 ap=108 tcp=156 topuser=madalice 2026-08-15 07:43:48 L1=27.28 run=20 blk=81 avail=259 (2.2%) swpO=42058 conn=85 myRSS=532 php=90 phpRSS=11353 ap=119 tcp=154 topuser=madalice 2026-08-15 07:44:28 L1=104.09 run=226 blk=2 avail=54 (0.5%) swpO=8286 conn=- myRSS=0 php=101 phpRSS=4346 ap=149 tcp=96 topuser=madalice 2026-08-15 07:44:26 L1=95.66 run=224 blk=30 avail=41 (0.4%) swpO=16065 conn=- myRSS=0 php=102 phpRSS=4390 ap=149 tcp=96 topuser=madalice 2026-08-15 07:50:51 L1=260.46 run=46 blk=2 avail=2932 (25.0%) swpO=166 conn=42 myRSS=475 php=40 phpRSS=7151 ap=135 tcp=19 topuser=madalice 2026-08-15 07:50:51 L1=260.46 run=42 blk=1 avail=2833 (24.2%) swpO=165 conn=41 myRSS=475 php=40 phpRSS=7250 ap=135 tcp=19 topuser=madalice