INCIDENT 2026-08-20 01:50:23 trigger: php_rss=1553MB ================================================================== load: 1.59 0.57 0.57 14/641 3270893 ncpu:10 mem_avail: 5455MB/11729MB (46.5%) swap_used:2462MB swapout:0/s blocked(D):0 running:12 iowait:0.0% steal:0.0% php: 10 procs / 1553MB heaviest user: madalice (10 procs, 1553MB) apache: 18 procs tcp_estab: 35 --- RSS by user (all processes) --- USER PROCS RSS_MB mysql 1 2755 madalice 11 1810 root 244 713 nobody 17 277 netdata 8 267 mailman 20 140 cpanellogin 1 64 evolution 7 61 kurri 4 46 dovenull 6 26 candyfabcom 2 23 wp-toolkit 6 19 mailnull 2 16 zabbix 1 15 named 1 15 aaa 1 12 lowrange 1 11 nscd 1 7 dovecot 3 6 cpanelconnecttrack 1 3 --- php workers by user --- USER PROCS RSS_MB madalice 11 1850 evolution 1 40 --- vm tuning / memory pressure --- vm.swappiness 30 vm.vfs_cache_pressure 100 vm.min_free_kbytes 67584 vm.overcommit_memory 0 vm.overcommit_ratio 50 Cached: 5425 MB Active(file): 3175 MB Inactive(file): 1340 MB Dirty: 0 MB Writeback: 0 MB SReclaimable: 632 MB CommitLimit: 9960 MB Committed_AS: 8633 MB --- backup / dump / scan processes (page-cache evictors) --- PID USER ELAPSED RSS_KB COMMAND 3248642 root 39:07 6376 /usr/bin/imunify-agent-proxy --- top 60 by RSS --- PID PPID USER RSS %CPU STAT ELAPSED COMMAND 2575700 1 mysql 2821132 10.8 Ssl 4-17:59:47 /usr/sbin/mariadbd 3270849 3270749 madalice 265068 106 R 00:02 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 3270850 3270817 madalice 254056 106 R 00:02 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 3270852 3269295 madalice 252496 105 R 00:02 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 3270853 3268073 madalice 246608 105 R 00:02 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 3270851 3267523 madalice 230436 99.5 R 00:02 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 1220 1 netdata 217056 2.5 Ssl 51-15:40:29 /usr/sbin/netdata -P /run/netdata/netdata.pid -D 3270854 3265082 madalice 200816 87.5 R 00:02 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 3270855 3267524 madalice 196596 174 R 00:01 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 3270856 3265060 madalice 183604 158 R 00:01 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 3270858 3270857 madalice 168384 140 R 00:01 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2661802 2112450 root 146368 0.0 S 12:36:08 spamd child 2575382 1 root 114132 0.0 Ss 4-17:59:50 /usr/lib/systemd/systemd-journald 3270860 3270714 madalice 109664 77.0 R 00:01 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 3270885 3270861 madalice 67720 0.0 R 00:00 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 3141574 1353308 cpanellogin 65888 0.0 S 01:50:14 /usr/local/cpanel/base/show_template.stor cpanel_locale docroot /usr/local/cpanel/base default_login_theme cpanel server_locale en 2112450 1 root 62936 0.0 Ss 23:26:51 /usr/local/cpanel/3rdparty/perl/542/bin/perl -T -w /usr/local/cpanel/3rdparty/bin/spamd --allowed-ips=127.0.0.1,::1 --max-children=5 -- 3255140 2112450 root 61488 0.0 S 29:37 spamd child 2223 1201 root 48068 0.0 Sl 51-15:40:28 /opt/saltstack/salt/bin/python3.10 /usr/bin/salt-minion MultiMinionProcessManager MinionProcessManager 3270879 3270862 evolution 41064 0.0 S 00:00 /opt/cpanel/ea-php74/root/usr/bin/php-cgi 1353308 1 root 40812 0.0 Ss 28-23:08:59 cpsrvd (SSL) - waiting for connections 4922 2431 netdata 33064 0.1 Sl 51-15:40:24 /usr/libexec/netdata/plugins.d/go.d.plugin 1 3141466 1 root 30792 0.0 Ss 01:50:22 lfd - sleeping 1469 1 root 28024 0.1 Ss 51-15:40:28 cpdavd - accepting connections on: 2079, 2080, 2091, 2077, 2078 3265060 1276780 nobody 25236 0.0 S 11:16 /usr/sbin/httpd -k start 3266627 1276780 nobody 24236 0.0 S 08:57 /usr/sbin/httpd -k start 3265082 1276780 nobody 21492 0.0 S 11:14 /usr/sbin/httpd -k start 3267525 1276780 nobody 19856 0.0 S 07:09 /usr/sbin/httpd -k start 3267523 1276780 nobody 19260 0.0 S 07:09 /usr/sbin/httpd -k start 3268073 1276780 nobody 19240 0.0 S 05:51 /usr/sbin/httpd -k start 3267524 1276780 nobody 19000 0.0 S 07:09 /usr/sbin/httpd -k start 3269295 1276780 nobody 18244 0.0 S 03:20 /usr/sbin/httpd -k start 185104 2431 netdata 17080 2.1 Sl 7-05:16:47 /usr/libexec/netdata/plugins.d/apps.plugin 1 3270747 1276780 nobody 16980 0.0 S 00:20 /usr/sbin/httpd -k start 1463 1 root 16896 0.0 Ss 51-15:40:28 tailwatchd 3262081 1471 root 16532 0.0 S 16:25 cPhulkd - dbprocessor 3270714 1276780 nobody 16424 0.0 S 00:21 /usr/sbin/httpd -k start 1204 1 zabbix 16188 0.2 Ssl 51-15:40:29 /usr/sbin/zabbix_agent2 -c /etc/zabbix/zabbix_agent2.conf 2107664 1 named 16036 0.0 Ssl 23:27:43 /usr/sbin/pdns_server --socket-dir=/run/pdns --guardian=no --daemon=no --disable-syslog --log-timestamp=no --write-pid=no 3270857 1276780 nobody 15896 0.0 S 00:01 /usr/sbin/httpd -k start 3270817 1276780 nobody 15828 0.0 S 00:08 /usr/sbin/httpd -k start 3270861 1276780 nobody 15820 0.0 S 00:00 /usr/sbin/httpd -k start 3270749 1276780 nobody 15764 0.0 S 00:20 /usr/sbin/httpd -k start 3270862 1276780 nobody 15672 0.0 S 00:00 /usr/sbin/httpd -k start 1471 1 root 15480 0.0 Ss 51-15:40:28 cPhulkd - processor 3270573 1471 root 13864 0.0 S 00:25 cPhulkd - processor - http socket 3270820 3828812 candyfabcom 12448 0.0 S 00:07 dovecot/imap 3036636 3828812 aaa 12376 0.0 S 03:54:55 dovecot/imap 3260509 3828812 lowrange 12236 0.0 S 19:03 dovecot/imap 3270859 3828812 kurri 12120 0.0 S 00:01 dovecot/imap 3270789 3828812 candyfabcom 12036 0.0 S 00:11 dovecot/imap 3269392 3828812 kurri 11920 0.0 S 03:01 dovecot/imap 3243738 3828812 kurri 11864 0.0 S 43:23 dovecot/imap 3261082 3828812 kurri 11812 0.0 S 18:24 dovecot/imap 1276780 1 root 11060 0.0 Ss 14-10:19:04 /usr/sbin/httpd -k start 3268454 3828812 root 11048 0.0 S 05:09 dovecot/auth -w 2107879 1 root 10668 0.0 Ss 23:27:42 queueprocd - waiting up to 60s to process a task 3270814 2111716 mailnull 9632 0.0 S 00:09 /usr/sbin/exim -ps -bd -q15m -oP /var/spool/exim/exim-daemon.pid 2111825 2111790 mailman 9424 0.0 S 23:27:20 /usr/bin/python3 /usr/local/cpanel/3rdparty/mailman/bin/qrunner --runner=RetryRunner:0:1 -s 2111817 2111790 mailman 8868 0.0 S 23:27:20 /usr/bin/python3 /usr/local/cpanel/3rdparty/mailman/bin/qrunner --runner=OutgoingRunner:1:8 -s --- processes in D state (uninterruptible IO) --- --- mariadb processlist (non-sleeping) --- +----------+------+------+---------+------+----------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ | ID | USER | DB | COMMAND | TIME | STATE | q | +----------+------+------+---------+------+----------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ | 14528618 | root | NULL | Query | 0 | Filling schema table | SELECT ID,USER,DB,COMMAND,TIME,STATE,LEFT(INFO,300) q FROM information_schema.PROCESSLIST WHERE COMMAND<>'Sleep' ORDER BY TIME DESC LIMIT 40 | +----------+------+------+---------+------+----------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ --- lock waits --- --- vhost traffic RIGHT NOW (last 3 min, deduped by inode) --- VHOST LOG REQ/3min whiz/whizengineering.com.au 130 evolution/portal.evolutionerp.com.au-ssl_log 19 coscom/coscom.com.au-ssl_log 2 madalice/madalice.com.au-ssl_log 2 (total vhosts taking traffic: 4) --- top client IPs across all vhosts (last 3 min) --- 130 20.25.139.174 19 203.54.180.238 2 20.171.51.14 1 45.148.10.125 1 35.229.51.64 --- top request paths across all vhosts (last 3 min) --- 19 GET /heartbeat.php 2 GET /wp-content/plugins/hellopress/wp_filemanager.php 2 GET /this_is_a_new_hello_world.php 2 GET /css.php 2 GET /cong.php 2 GET /alfa.php 2 GET /admin.php 2 GET /404.php 1 GET /zwso.php 1 GET /z60.php 1 GET /xx.php 1 GET /xv.php 1 GET /x.php 1 GET /x56.php 1 GET /x50.php 1 GET /ww1.php 1 GET /wsd.php 1 GET /wrt.php 1 GET /wp-trackback.php 1 GET /wp-includes/style-engine/about.php --- busiest 3 vhosts: top paths + last 10 requests --- == whiz/whizengineering.com.au == -- top paths -- 2 GET /css.php 2 GET /cong.php 2 GET /alfa.php 2 GET /admin.php 2 GET /404.php 1 GET /zwso.php 1 GET /z60.php 1 GET /xx.php 1 GET /xv.php 1 GET /x.php 1 GET /x56.php 1 GET /x50.php -- last 10 requests -- 20.25.139.174 - - [20/Aug/2026:01:49:51 +1000] "GET /ave.php HTTP/1.1" 404 - "-" "-" 20.25.139.174 - - [20/Aug/2026:01:49:51 +1000] "GET /wrt.php HTTP/1.1" 404 - "-" "-" 20.25.139.174 - - [20/Aug/2026:01:49:51 +1000] "GET /bibil.php HTTP/1.1" 404 - "-" "-" 20.25.139.174 - - [20/Aug/2026:01:49:52 +1000] "GET /radio.php HTTP/1.1" 403 13 "-" "-" 20.25.139.174 - - [20/Aug/2026:01:49:52 +1000] "GET /files.php HTTP/1.1" 404 - "-" "-" 20.25.139.174 - - [20/Aug/2026:01:49:52 +1000] "GET /file7.php HTTP/1.1" 404 - "-" "-" 20.25.139.174 - - [20/Aug/2026:01:49:52 +1000] "GET /gifclass.php HTTP/1.1" 404 - "-" "-" 20.25.139.174 - - [20/Aug/2026:01:49:52 +1000] "GET /wp-admin/css/colors/midnight/about.php HTTP/1.1" 404 - "-" "-" 20.25.139.174 - - [20/Aug/2026:01:49:52 +1000] "GET /nox.php HTTP/1.1" 404 - "-" "-" 20.25.139.174 - - [20/Aug/2026:01:49:53 +1000] "GET /file48.php HTTP/1.1" 404 - "-" "-" == evolution/portal.evolutionerp.com.au-ssl_log == -- top paths -- 19 GET /heartbeat.php -- last 10 requests -- 203.54.180.238 - - [20/Aug/2026:01:48:42 +1000] "GET /heartbeat.php?sentAt=1787154523&bytes=106,133,208,91&_=1787115832319 HTTP/1.1" 200 10 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6" 203.54.180.238 - - [20/Aug/2026:01:48:51 +1000] "GET /heartbeat.php?sentAt=1787154531&bytes=106,133,208,99&_=1787143288244 HTTP/1.1" 200 10 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6" 203.54.180.238 - - [20/Aug/2026:01:48:52 +1000] "GET /heartbeat.php?sentAt=1787154533&bytes=106,133,208,101&_=1787115832320 HTTP/1.1" 200 10 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6 203.54.180.238 - - [20/Aug/2026:01:49:01 +1000] "GET /heartbeat.php?sentAt=1787154541&bytes=106,133,208,109&_=1787143288245 HTTP/1.1" 200 10 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6 203.54.180.238 - - [20/Aug/2026:01:49:02 +1000] "GET /heartbeat.php?sentAt=1787154543&bytes=106,133,208,111&_=1787115832321 HTTP/1.1" 200 10 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6 203.54.180.238 - - [20/Aug/2026:01:49:11 +1000] "GET /heartbeat.php?sentAt=1787154551&bytes=106,133,208,119&_=1787143288246 HTTP/1.1" 200 10 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6 203.54.180.238 - - [20/Aug/2026:01:49:12 +1000] "GET /heartbeat.php?sentAt=1787154553&bytes=106,133,208,121&_=1787115832322 HTTP/1.1" 200 10 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6 203.54.180.238 - - [20/Aug/2026:01:49:21 +1000] "GET /heartbeat.php?sentAt=1787154561&bytes=106,133,208,129&_=1787143288247 HTTP/1.1" 200 10 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6 203.54.180.238 - - [20/Aug/2026:01:49:22 +1000] "GET /heartbeat.php?sentAt=1787154563&bytes=106,133,208,131&_=1787115832323 HTTP/1.1" 200 10 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6 203.54.180.238 - - [20/Aug/2026:01:49:31 +1000] "GET /heartbeat.php?sentAt=1787154571&bytes=106,133,208,139&_=1787143288248 HTTP/1.1" 200 10 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6 == coscom/coscom.com.au-ssl_log == -- top paths -- 1 GET /wp-content/plugins/hellopress/wp_filemanager.php 1 GET /this_is_a_new_hello_world.php -- last 10 requests -- 20.171.51.14 - - [20/Aug/2026:01:49:37 +1000] "GET /wp-content/plugins/hellopress/wp_filemanager.php HTTP/1.1" 404 5750 "-" "-" 20.171.51.14 - - [20/Aug/2026:01:49:38 +1000] "GET /this_is_a_new_hello_world.php HTTP/1.1" 404 5750 "-" "-" --- last 30 samples (run-up to this incident) --- 2026-08-20 01:47:57 L1=0.32 run=1 blk=0 avail=6684 (57.0%) swpO=0 conn=1 myRSS=2755 php=0 phpRSS=0 ap=14 tcp=21 topuser=- 2026-08-20 01:48:03 L1=0.29 run=1 blk=0 avail=6669 (56.9%) swpO=0 conn=2 myRSS=2755 php=1 phpRSS=41 ap=14 tcp=23 topuser=evolution 2026-08-20 01:48:07 L1=0.35 run=1 blk=0 avail=6682 (57.0%) swpO=0 conn=1 myRSS=2755 php=0 phpRSS=0 ap=14 tcp=22 topuser=- 2026-08-20 01:48:13 L1=0.40 run=1 blk=0 avail=6674 (56.9%) swpO=0 conn=2 myRSS=2755 php=1 phpRSS=40 ap=14 tcp=21 topuser=evolution 2026-08-20 01:48:17 L1=0.37 run=1 blk=0 avail=6686 (57.0%) swpO=0 conn=1 myRSS=2755 php=0 phpRSS=0 ap=14 tcp=21 topuser=- 2026-08-20 01:48:23 L1=0.34 run=1 blk=0 avail=6668 (56.9%) swpO=0 conn=2 myRSS=2755 php=1 phpRSS=41 ap=14 tcp=23 topuser=evolution 2026-08-20 01:48:27 L1=0.31 run=3 blk=0 avail=6680 (57.0%) swpO=0 conn=1 myRSS=2755 php=0 phpRSS=0 ap=14 tcp=24 topuser=- 2026-08-20 01:48:33 L1=0.29 run=2 blk=0 avail=6554 (55.9%) swpO=0 conn=2 myRSS=2755 php=2 phpRSS=188 ap=14 tcp=27 topuser=speedlin 2026-08-20 01:48:37 L1=0.26 run=1 blk=0 avail=6680 (57.0%) swpO=0 conn=1 myRSS=2755 php=0 phpRSS=0 ap=14 tcp=24 topuser=- 2026-08-20 01:48:43 L1=0.24 run=1 blk=0 avail=6667 (56.8%) swpO=0 conn=2 myRSS=2755 php=1 phpRSS=41 ap=14 tcp=25 topuser=evolution 2026-08-20 01:48:47 L1=0.22 run=1 blk=0 avail=6678 (56.9%) swpO=0 conn=1 myRSS=2755 php=0 phpRSS=0 ap=14 tcp=25 topuser=- 2026-08-20 01:48:53 L1=0.20 run=1 blk=0 avail=6670 (56.9%) swpO=0 conn=2 myRSS=2755 php=1 phpRSS=41 ap=14 tcp=24 topuser=evolution 2026-08-20 01:48:57 L1=0.19 run=1 blk=0 avail=6684 (57.0%) swpO=0 conn=1 myRSS=2755 php=0 phpRSS=0 ap=14 tcp=23 topuser=- 2026-08-20 01:49:03 L1=0.17 run=2 blk=0 avail=6461 (55.1%) swpO=0 conn=2 myRSS=2755 php=2 phpRSS=274 ap=14 tcp=27 topuser=corpkicks 2026-08-20 01:49:07 L1=0.16 run=1 blk=0 avail=6675 (56.9%) swpO=0 conn=1 myRSS=2755 php=0 phpRSS=0 ap=14 tcp=27 topuser=- 2026-08-20 01:49:13 L1=0.14 run=1 blk=0 avail=6676 (56.9%) swpO=0 conn=2 myRSS=2755 php=1 phpRSS=41 ap=14 tcp=23 topuser=evolution 2026-08-20 01:49:17 L1=0.13 run=1 blk=0 avail=6690 (57.0%) swpO=0 conn=1 myRSS=2755 php=0 phpRSS=0 ap=14 tcp=24 topuser=- 2026-08-20 01:49:23 L1=0.12 run=1 blk=0 avail=6676 (56.9%) swpO=0 conn=2 myRSS=2755 php=1 phpRSS=41 ap=14 tcp=25 topuser=evolution 2026-08-20 01:49:27 L1=0.11 run=1 blk=0 avail=6688 (57.0%) swpO=0 conn=1 myRSS=2755 php=0 phpRSS=0 ap=14 tcp=22 topuser=- 2026-08-20 01:49:33 L1=0.10 run=1 blk=0 avail=6674 (56.9%) swpO=0 conn=2 myRSS=2755 php=1 phpRSS=41 ap=14 tcp=25 topuser=evolution 2026-08-20 01:49:37 L1=0.18 run=1 blk=0 avail=6684 (57.0%) swpO=0 conn=1 myRSS=2755 php=0 phpRSS=0 ap=14 tcp=27 topuser=- 2026-08-20 01:49:43 L1=0.16 run=1 blk=0 avail=6675 (56.9%) swpO=0 conn=2 myRSS=2755 php=1 phpRSS=41 ap=14 tcp=29 topuser=evolution 2026-08-20 01:49:47 L1=0.15 run=1 blk=0 avail=6685 (57.0%) swpO=0 conn=1 myRSS=2755 php=0 phpRSS=0 ap=14 tcp=27 topuser=- 2026-08-20 01:49:53 L1=0.14 run=2 blk=0 avail=6672 (56.9%) swpO=0 conn=2 myRSS=2755 php=1 phpRSS=41 ap=14 tcp=26 topuser=evolution 2026-08-20 01:49:57 L1=0.12 run=1 blk=0 avail=6694 (57.1%) swpO=0 conn=1 myRSS=2755 php=0 phpRSS=0 ap=14 tcp=17 topuser=- 2026-08-20 01:50:03 L1=0.11 run=2 blk=0 avail=6535 (55.7%) swpO=0 conn=2 myRSS=2755 php=2 phpRSS=187 ap=17 tcp=27 topuser=alison 2026-08-20 01:50:07 L1=0.10 run=1 blk=0 avail=6666 (56.8%) swpO=0 conn=1 myRSS=2755 php=0 phpRSS=0 ap=19 tcp=20 topuser=- 2026-08-20 01:50:13 L1=0.10 run=1 blk=0 avail=6679 (56.9%) swpO=0 conn=2 myRSS=2755 php=1 phpRSS=41 ap=14 tcp=21 topuser=evolution 2026-08-20 01:50:17 L1=0.09 run=1 blk=0 avail=6683 (57.0%) swpO=0 conn=1 myRSS=2755 php=0 phpRSS=0 ap=15 tcp=24 topuser=- 2026-08-20 01:50:23 L1=0.16 run=12 blk=0 avail=5455 (46.5%) swpO=0 conn=12 myRSS=2755 php=10 phpRSS=1553 ap=18 tcp=35 topuser=madalice