INCIDENT 2026-08-15 07:13:10 trigger: php_rss=1534MB ================================================================== load: 1.52 7.86 12.21 18/643 2553392 ncpu:10 mem_avail: 7964MB/11729MB (67.9%) swap_used:1323MB swapout:0/s blocked(D):0 running:17 iowait:0.1% steal:0.0% php: 13 procs / 1534MB heaviest user: madalice (11 procs, 1102MB) apache: 50 procs tcp_estab: 41 --- RSS by user (all processes) --- USER PROCS RSS_MB madalice 12 1311 nobody 49 614 mysql 1 564 root 234 541 netdata 8 229 corpkicks 1 223 cpanellogin 1 89 dovenull 6 20 mailman 20 19 wp-toolkit 6 18 zabbix 1 12 kamcool 1 11 cpanelphpmyadmin 1 8 kurri 3 7 dovecot 3 7 named 1 4 cpanelconnecttrack 1 3 polkitd 1 2 mailnull 1 2 evolution 4 2 --- php workers by user --- USER PROCS RSS_MB madalice 13 1357 corpkicks 1 227 --- vm tuning / memory pressure --- vm.swappiness 30 vm.vfs_cache_pressure 100 vm.min_free_kbytes 67584 vm.overcommit_memory 0 vm.overcommit_ratio 50 Cached: 2934 MB Active(file): 1257 MB Inactive(file): 1114 MB Dirty: 0 MB Writeback: 0 MB SReclaimable: 210 MB CommitLimit: 9960 MB Committed_AS: 9286 MB --- backup / dump / scan processes (page-cache evictors) --- PID USER ELAPSED RSS_KB COMMAND 2342983 root 06:01:55 208 /usr/bin/imunify-agent-proxy 2550360 root 03:56 64408 /opt/imunify360/venv/bin/python3 -m imav.run --- top 60 by RSS --- PID PPID USER RSS %CPU STAT ELAPSED COMMAND 2544865 1 mysql 578464 4.0 Ssl 12:11 /usr/sbin/mariadbd 2553335 2550045 madalice 183620 78.0 R 00:02 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2553334 2553296 madalice 175228 77.0 R 00:02 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 1220 1 netdata 173768 2.5 Ssl 46-21:03:13 /usr/sbin/netdata -P /run/netdata/netdata.pid -D 2553336 2551105 madalice 169340 74.0 R 00:02 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2553337 2553321 madalice 160104 66.5 R 00:02 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2553340 2553322 madalice 158020 130 S 00:01 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2390233 2382613 root 146840 0.0 S 04:37:33 spamd child 2553341 2553324 madalice 126680 98.0 R 00:01 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2553342 2553326 madalice 125852 98.0 R 00:01 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2553343 2553327 madalice 109916 79.0 R 00:01 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2553360 2553329 madalice 106768 76.0 R 00:01 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2525318 1353308 cpanellogin 91144 0.0 S 44:58 /usr/local/cpanel/base/show_template.stor docroot /usr/local/cpanel/base cpanel_locale default_login_theme cpanel server_locale en 2382613 1 root 82984 0.0 Ss 04:49:59 /usr/local/cpanel/3rdparty/perl/542/bin/perl -T -w /usr/local/cpanel/3rdparty/bin/spamd --allowed-ips=127.0.0.1,::1 --max-children=5 -- 2553362 2553331 madalice 76056 34.0 R 00:01 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2553361 2553330 madalice 75312 34.0 R 00:01 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2553379 2553332 madalice 72600 0.0 R 00:00 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2550360 1 root 64408 0.5 Ssl 03:56 /opt/imunify360/venv/bin/python3 -m imav.run 159820 1 root 59424 0.0 Ss 25-21:26:21 /usr/lib/systemd/systemd-journald 2553397 2550998 madalice 58188 0.0 R 00:00 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2553402 2553344 madalice 54248 0.0 R 00:00 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2553409 2553349 madalice 49484 0.0 R 00:00 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 2553417 2551019 madalice 43172 0.0 R 00:00 /opt/cpanel/ea-php82/root/usr/bin/php-cgi 4922 2431 netdata 42688 0.1 Sl 46-21:03:08 /usr/libexec/netdata/plugins.d/go.d.plugin 1 2223 1201 root 26416 0.0 Sl 46-21:03:12 /opt/saltstack/salt/bin/python3.10 /usr/bin/salt-minion MultiMinionProcessManager MinionProcessManager 2390234 2382613 root 22324 0.0 S 04:37:33 spamd child 2550998 1276780 nobody 21408 0.0 S 02:59 /usr/sbin/httpd -k start 1469 1 root 21156 0.1 Ss 46-21:03:12 cpdavd - accepting connections on: 2079, 2080, 2091, 2077, 2078 2548751 1276780 nobody 20328 0.0 S 05:55 /usr/sbin/httpd -k start 2551019 1276780 nobody 20280 0.0 S 02:58 /usr/sbin/httpd -k start 2550045 1276780 nobody 19892 0.0 S 04:28 /usr/sbin/httpd -k start 2381704 1 root 19844 0.0 Ss 04:50:58 lfd - sleeping 2551277 1276780 nobody 19588 0.0 S 02:53 /usr/sbin/httpd -k start 2551021 1276780 nobody 19512 0.0 S 02:58 /usr/sbin/httpd -k start 2551015 1276780 nobody 19364 0.0 S 02:58 /usr/sbin/httpd -k start 2551341 1276780 nobody 18948 0.0 S 02:51 /usr/sbin/httpd -k start 2551437 1276780 nobody 18540 0.0 S 02:45 /usr/sbin/httpd -k start 2551363 1276780 nobody 17984 0.0 S 02:50 /usr/sbin/httpd -k start 2551357 1276780 nobody 17588 0.0 S 02:50 /usr/sbin/httpd -k start 2551105 1276780 nobody 17468 0.0 S 02:55 /usr/sbin/httpd -k start 2551360 1276780 nobody 17264 0.0 S 02:50 /usr/sbin/httpd -k start 2553290 1276780 nobody 17176 0.2 S 00:05 /usr/sbin/httpd -k start 2551480 1276780 nobody 17088 0.0 S 02:44 /usr/sbin/httpd -k start 2553328 1276780 nobody 16908 0.5 S 00:02 /usr/sbin/httpd -k start 2553333 1276780 nobody 16780 0.0 S 00:02 /usr/sbin/httpd -k start 185104 2431 netdata 16136 2.0 Sl 2-10:39:31 /usr/libexec/netdata/plugins.d/apps.plugin 1 2552523 1276780 nobody 15948 0.0 S 01:02 /usr/sbin/httpd -k start 2553323 1276780 nobody 15828 0.0 S 00:03 /usr/sbin/httpd -k start 2553295 1276780 nobody 15376 0.2 S 00:04 /usr/sbin/httpd -k start 2552464 1276780 nobody 15308 0.0 S 01:07 /usr/sbin/httpd -k start 2553332 1276780 nobody 15136 0.5 S 00:02 /usr/sbin/httpd -k start 2553321 1276780 nobody 15112 0.0 S 00:03 /usr/sbin/httpd -k start 2553326 1276780 nobody 15088 0.5 S 00:02 /usr/sbin/httpd -k start 2553330 1276780 nobody 15064 0.0 S 00:02 /usr/sbin/httpd -k start 2553324 1276780 nobody 15044 0.0 S 00:03 /usr/sbin/httpd -k start 2553329 1276780 nobody 15044 0.0 S 00:02 /usr/sbin/httpd -k start 2553344 1276780 nobody 15040 0.0 S 00:01 /usr/sbin/httpd -k start 2553349 1276780 nobody 15004 0.0 S 00:01 /usr/sbin/httpd -k start 2553296 1276780 nobody 14984 0.0 S 00:04 /usr/sbin/httpd -k start 2553327 1276780 nobody 14980 0.5 S 00:02 /usr/sbin/httpd -k start --- processes in D state (uninterruptible IO) --- --- mariadb processlist (non-sleeping) --- +------+------+------+---------+------+----------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ | ID | USER | DB | COMMAND | TIME | STATE | q | +------+------+------+---------+------+----------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ | 1024 | root | NULL | Query | 0 | Filling schema table | SELECT ID,USER,DB,COMMAND,TIME,STATE,LEFT(INFO,300) q FROM information_schema.PROCESSLIST WHERE COMMAND<>'Sleep' ORDER BY TIME DESC LIMIT 40 | +------+------+------+---------+------+----------------------+------------------------------------------------------------------------------------------------------------------------------------------------------------+ --- lock waits --- --- vhost traffic RIGHT NOW (last 3 min, deduped by inode) --- VHOST LOG REQ/3min olytis/tis.com.au-ssl_log 91 otterfencing/otterfencing.com.au-ssl_log 41 evolution/portal.evolutionerp.com.au-ssl_log 25 madalice/madalice.com.au-ssl_log 2 evolution/my.evolutionerp.com.au-ssl_log 1 (total vhosts taking traffic: 5) --- top client IPs across all vhosts (last 3 min) --- 91 20.65.105.233 41 68.221.130.146 25 203.54.180.238 2 216.73.216.26 1 40.77.167.78 --- top request paths across all vhosts (last 3 min) --- 25 GET /heartbeat.php 2 GET /xx.php 2 GET /wp-content/plugins/hellopress/wp_filemanager.php 2 GET /this_is_a_new_hello_world.php 2 GET /oxvrws.php 2 GET /k.php 2 GET /1wx.php 1 GET /xxxx.php 1 GET /xxw.php 1 GET /x7.php 1 GET /wto.php 1 GET /wsd.php 1 GET /ws45.php 1 GET /ws38.php 1 GET /ws37.php 1 GET /ws36.php 1 GET /ws34.php 1 GET /ws33.php 1 GET /wfile.php 1 GET /wcas.php --- busiest 3 vhosts: top paths + last 10 requests --- == olytis/tis.com.au-ssl_log == -- top paths -- 1 GET /xxxx.php 1 GET /xxw.php 1 GET /xx.php 1 GET /x7.php 1 GET /wto.php 1 GET /wsd.php 1 GET /ws45.php 1 GET /ws38.php 1 GET /ws37.php 1 GET /ws36.php 1 GET /ws34.php 1 GET /ws33.php -- last 10 requests -- 20.65.105.233 - - [15/Aug/2026:07:12:28 +1000] "GET /class629.php HTTP/1.1" 404 5750 "-" "-" 20.65.105.233 - - [15/Aug/2026:07:12:28 +1000] "GET /file9.php HTTP/1.1" 404 5750 "-" "-" 20.65.105.233 - - [15/Aug/2026:07:12:28 +1000] "GET /moon.php HTTP/1.1" 404 5750 "-" "-" 20.65.105.233 - - [15/Aug/2026:07:12:29 +1000] "GET /file18.php HTTP/1.1" 404 5750 "-" "-" 20.65.105.233 - - [15/Aug/2026:07:12:29 +1000] "GET /file21.php HTTP/1.1" 404 5750 "-" "-" 20.65.105.233 - - [15/Aug/2026:07:12:30 +1000] "GET /xxw.php HTTP/1.1" 404 5750 "-" "-" 20.65.105.233 - - [15/Aug/2026:07:12:30 +1000] "GET /racs3.php HTTP/1.1" 404 5750 "-" "-" 20.65.105.233 - - [15/Aug/2026:07:12:30 +1000] "GET /wsd.php HTTP/1.1" 404 5750 "-" "-" 20.65.105.233 - - [15/Aug/2026:07:12:31 +1000] "GET /333.php HTTP/1.1" 404 5750 "-" "-" 20.65.105.233 - - [15/Aug/2026:07:12:31 +1000] "GET /12.php HTTP/1.1" 404 5750 "-" "-" == otterfencing/otterfencing.com.au-ssl_log == -- top paths -- 2 GET /oxvrws.php 2 GET /1wx.php 1 GET /xx.php 1 GET /wp-content/plugins/hellopress/wp_filemanager.php 1 GET /up.php 1 GET /uplozyu.php 1 GET /uploxe.php 1 GET /uploat.php 1 GET /this_is_a_new_hello_world.php 1 GET /system.php 1 GET /sts.php 1 GET /shell20211028.php -- last 10 requests -- 68.221.130.146 - - [15/Aug/2026:07:12:57 +1000] "GET /oxvrws.php HTTP/1.1" 401 36 "-" "-" 68.221.130.146 - - [15/Aug/2026:07:12:57 +1000] "GET /sts.php HTTP/1.1" 401 36 "-" "-" 68.221.130.146 - - [15/Aug/2026:07:12:58 +1000] "GET /system.php HTTP/1.1" 401 36 "-" "-" 68.221.130.146 - - [15/Aug/2026:07:12:59 +1000] "GET /layout.php HTTP/1.1" 401 36 "-" "-" 68.221.130.146 - - [15/Aug/2026:07:12:59 +1000] "GET /1wx.php HTTP/1.1" 401 36 "-" "-" 68.221.130.146 - - [15/Aug/2026:07:13:00 +1000] "GET /BbiNip.php HTTP/1.1" 401 36 "-" "-" 68.221.130.146 - - [15/Aug/2026:07:13:00 +1000] "GET /1wx.php HTTP/1.1" 401 36 "-" "-" 68.221.130.146 - - [15/Aug/2026:07:13:01 +1000] "GET /lzmelu.php HTTP/1.1" 401 36 "-" "-" 68.221.130.146 - - [15/Aug/2026:07:13:01 +1000] "GET /up.php HTTP/1.1" 401 36 "-" "-" 68.221.130.146 - - [15/Aug/2026:07:13:02 +1000] "GET /Hd.php HTTP/1.1" 401 36 "-" "-" == evolution/portal.evolutionerp.com.au-ssl_log == -- top paths -- 25 GET /heartbeat.php -- last 10 requests -- 203.54.180.238 - - [15/Aug/2026:07:12:15 +1000] "GET /heartbeat.php?sentAt=1786741936&bytes=106,127,132,176&_=1786712385211 HTTP/1.1" 200 10 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6 203.54.180.238 - - [15/Aug/2026:07:12:23 +1000] "GET /heartbeat.php?sentAt=1786741943&bytes=106,127,132,183&_=1786683795244 HTTP/1.1" 200 10 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6 203.54.180.238 - - [15/Aug/2026:07:12:25 +1000] "GET /heartbeat.php?sentAt=1786741946&bytes=106,127,132,186&_=1786712385212 HTTP/1.1" 200 10 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6 203.54.180.238 - - [15/Aug/2026:07:12:33 +1000] "GET /heartbeat.php?sentAt=1786741953&bytes=106,127,132,193&_=1786683795245 HTTP/1.1" 200 10 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6 203.54.180.238 - - [15/Aug/2026:07:12:35 +1000] "GET /heartbeat.php?sentAt=1786741956&bytes=106,127,132,196&_=1786712385213 HTTP/1.1" 200 10 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6 203.54.180.238 - - [15/Aug/2026:07:12:43 +1000] "GET /heartbeat.php?sentAt=1786741963&bytes=106,127,132,203&_=1786683795246 HTTP/1.1" 200 10 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6 203.54.180.238 - - [15/Aug/2026:07:12:45 +1000] "GET /heartbeat.php?sentAt=1786741966&bytes=106,127,132,206&_=1786712385214 HTTP/1.1" 200 10 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6 203.54.180.238 - - [15/Aug/2026:07:12:53 +1000] "GET /heartbeat.php?sentAt=1786741973&bytes=106,127,132,213&_=1786683795247 HTTP/1.1" 200 10 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6 203.54.180.238 - - [15/Aug/2026:07:12:55 +1000] "GET /heartbeat.php?sentAt=1786741976&bytes=106,127,132,216&_=1786712385215 HTTP/1.1" 200 10 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6 203.54.180.238 - - [15/Aug/2026:07:13:03 +1000] "GET /heartbeat.php?sentAt=1786741983&bytes=106,127,132,223&_=1786683795248 HTTP/1.1" 200 10 "https://portal.evolutionerp.com.au/index.php?page=factory_proc_queue&process=6 --- last 30 samples (run-up to this incident) --- 2026-08-15 07:10:46 L1=9.56 run=6 blk=1 avail=8183 (69.8%) swpO=0 conn=39 myRSS=559 php=7 phpRSS=862 ap=72 tcp=31 topuser=madalice 2026-08-15 07:10:49 L1=9.20 run=5 blk=0 avail=8571 (73.1%) swpO=0 conn=46 myRSS=561 php=4 phpRSS=412 ap=69 tcp=29 topuser=olympic 2026-08-15 07:10:56 L1=8.78 run=2 blk=0 avail=8806 (75.1%) swpO=0 conn=39 myRSS=563 php=2 phpRSS=164 ap=62 tcp=29 topuser=madalice 2026-08-15 07:10:59 L1=8.16 run=2 blk=0 avail=8919 (76.0%) swpO=0 conn=46 myRSS=563 php=0 phpRSS=0 ap=59 tcp=24 topuser=- 2026-08-15 07:11:06 L1=7.51 run=1 blk=0 avail=8957 (76.4%) swpO=0 conn=39 myRSS=563 php=1 phpRSS=40 ap=52 tcp=20 topuser=evolution 2026-08-15 07:11:09 L1=6.90 run=1 blk=0 avail=8992 (76.7%) swpO=0 conn=1 myRSS=563 php=0 phpRSS=0 ap=49 tcp=21 topuser=- 2026-08-15 07:11:16 L1=6.35 run=1 blk=0 avail=9031 (77.0%) swpO=0 conn=39 myRSS=563 php=1 phpRSS=40 ap=42 tcp=22 topuser=evolution 2026-08-15 07:11:19 L1=5.84 run=1 blk=0 avail=9066 (77.3%) swpO=0 conn=1 myRSS=563 php=0 phpRSS=0 ap=39 tcp=17 topuser=- 2026-08-15 07:11:26 L1=5.37 run=1 blk=0 avail=9108 (77.6%) swpO=0 conn=2 myRSS=563 php=1 phpRSS=40 ap=32 tcp=17 topuser=evolution 2026-08-15 07:11:29 L1=4.94 run=1 blk=0 avail=9147 (78.0%) swpO=0 conn=1 myRSS=563 php=0 phpRSS=0 ap=29 tcp=16 topuser=- 2026-08-15 07:11:36 L1=4.55 run=1 blk=0 avail=9189 (78.3%) swpO=0 conn=2 myRSS=563 php=1 phpRSS=40 ap=22 tcp=20 topuser=evolution 2026-08-15 07:11:39 L1=4.18 run=1 blk=0 avail=9200 (78.4%) swpO=0 conn=1 myRSS=563 php=0 phpRSS=0 ap=21 tcp=22 topuser=- 2026-08-15 07:11:46 L1=3.85 run=2 blk=0 avail=9180 (78.3%) swpO=0 conn=2 myRSS=563 php=1 phpRSS=40 ap=21 tcp=20 topuser=evolution 2026-08-15 07:11:49 L1=3.54 run=1 blk=0 avail=9193 (78.4%) swpO=0 conn=1 myRSS=563 php=0 phpRSS=0 ap=21 tcp=22 topuser=- 2026-08-15 07:11:56 L1=3.26 run=1 blk=0 avail=9175 (78.2%) swpO=0 conn=2 myRSS=563 php=1 phpRSS=40 ap=21 tcp=25 topuser=evolution 2026-08-15 07:12:00 L1=2.99 run=1 blk=0 avail=9191 (78.4%) swpO=0 conn=1 myRSS=563 php=0 phpRSS=0 ap=21 tcp=29 topuser=- 2026-08-15 07:12:06 L1=2.75 run=1 blk=0 avail=9172 (78.2%) swpO=0 conn=2 myRSS=563 php=1 phpRSS=40 ap=22 tcp=27 topuser=evolution 2026-08-15 07:12:10 L1=2.61 run=1 blk=0 avail=9178 (78.2%) swpO=0 conn=1 myRSS=563 php=0 phpRSS=0 ap=23 tcp=23 topuser=- 2026-08-15 07:12:16 L1=2.48 run=3 blk=0 avail=9116 (77.7%) swpO=0 conn=2 myRSS=563 php=1 phpRSS=79 ap=23 tcp=24 topuser=madalice 2026-08-15 07:12:20 L1=2.29 run=1 blk=0 avail=9170 (78.2%) swpO=0 conn=1 myRSS=564 php=0 phpRSS=0 ap=23 tcp=22 topuser=- 2026-08-15 07:12:26 L1=2.10 run=3 blk=0 avail=9169 (78.2%) swpO=0 conn=1 myRSS=564 php=0 phpRSS=0 ap=23 tcp=25 topuser=- 2026-08-15 07:12:30 L1=1.93 run=1 blk=0 avail=9179 (78.3%) swpO=0 conn=1 myRSS=564 php=0 phpRSS=0 ap=22 tcp=20 topuser=- 2026-08-15 07:12:36 L1=1.78 run=2 blk=0 avail=9169 (78.2%) swpO=0 conn=1 myRSS=564 php=1 phpRSS=59 ap=20 tcp=20 topuser=alison 2026-08-15 07:12:40 L1=1.72 run=3 blk=0 avail=9011 (76.8%) swpO=0 conn=1 myRSS=564 php=1 phpRSS=207 ap=20 tcp=19 topuser=madalice 2026-08-15 07:12:46 L1=1.58 run=1 blk=0 avail=9203 (78.5%) swpO=0 conn=1 myRSS=564 php=0 phpRSS=0 ap=19 tcp=19 topuser=- 2026-08-15 07:12:50 L1=1.45 run=1 blk=0 avail=9200 (78.4%) swpO=0 conn=1 myRSS=564 php=0 phpRSS=0 ap=19 tcp=18 topuser=- 2026-08-15 07:12:57 L1=1.33 run=2 blk=0 avail=9200 (78.4%) swpO=0 conn=1 myRSS=564 php=0 phpRSS=0 ap=19 tcp=16 topuser=- 2026-08-15 07:13:00 L1=1.23 run=2 blk=0 avail=9197 (78.4%) swpO=0 conn=1 myRSS=564 php=0 phpRSS=0 ap=19 tcp=18 topuser=- 2026-08-15 07:13:07 L1=1.13 run=3 blk=0 avail=9127 (77.8%) swpO=0 conn=1 myRSS=564 php=2 phpRSS=124 ap=22 tcp=27 topuser=madalice 2026-08-15 07:13:10 L1=1.52 run=17 blk=0 avail=7964 (67.9%) swpO=0 conn=13 myRSS=564 php=13 phpRSS=1534 ap=50 tcp=41 topuser=madalice