How to Investigate a Compromised Linux Server: IR Playbook

Run these commands IMMEDIATELY: before any other action:

# 1. Capture system information snapshot:
date && hostname && uname -a > /tmp/snapshot.txt

# 2. Who is currently logged in:
w                        # Current sessions with IP and activity
last -100               # Last 100 logins (reads /var/log/wtmp)
lastb -100              # Failed logins (/var/log/btmp)
who -a                  # All logged-in users with TTY

# 3. Running processes:
ps auxf                 # All processes with full args, in tree format
# Look for: unusual process names, high CPU/memory, unexpected users
# pay attention to: www-data, apache, nginx spawning shells

# 4. Process network connections:
ss -tnpu                # All TCP/UDP connections with process info
# or:
netstat -tnpu
# Look for: ESTABLISHED connections to external IPs from unexpected processes
# Look for: LISTEN on unusual ports (backdoors typically listen on high ports)

# 5. Network statistics:
ss -s                   # Socket summary
ip route show           # Routing table (detect unauthorized routes added)
arp -a                  # ARP table (detect lateral movement neighbors)

# 6. Active network connections WITH process names and PIDs:
ss -tnpO
# Save to file:
ss -tnpO > /tmp/network_connections_$(date +%s).txt

# 7. List all open files and network sockets per process:
lsof -i -n -P          # All open network files (no DNS resolution)
lsof -u root           # All files opened by root

# 8. Capture process list with hashes (before any cleanup):
for pid in $(ls /proc/ | grep -E '^[0-9]+$'); do
    exe=$(readlink -f /proc/$pid/exe 2>/dev/null)
    cmd=$(cat /proc/$pid/cmdline 2>/dev/null | tr '\0' ' ')
    if [ -n "$exe" ]; then
        hash=$(sha256sum "$exe" 2>/dev/null | cut -d' ' -f1)
        echo "PID:$pid EXE:$exe HASH:$hash CMD:$cmd"
    fi
done > /tmp/process_hashes.txt

Examine all persistence locations: attackers commonly use multiple:

# SSH authorized keys (most common persistence):
for user in $(cut -d: -f1 /etc/passwd); do
    home=$(getent passwd $user | cut -d: -f6)
    keyfile="$home/.ssh/authorized_keys"
    if [ -f "$keyfile" ]; then
        echo "=== $user: $keyfile ==="
        ls -la "$keyfile"
        cat "$keyfile"
    fi
done
# Also check /root/.ssh/authorized_keys directly:
cat /root/.ssh/authorized_keys 2>/dev/null

# Cron jobs (all users and system):
crontab -l -u root 2>/dev/null
for user in $(cut -d: -f1 /etc/passwd); do
    crontab -l -u $user 2>/dev/null && echo "--- for $user ---"
done
cat /etc/crontab
ls -la /etc/cron.d/ /etc/cron.hourly/ /etc/cron.daily/ /etc/cron.weekly/
cat /etc/cron.d/* 2>/dev/null

# Systemd units (persistent service-based backdoors):
find /etc/systemd /lib/systemd /usr/lib/systemd -name "*.service" \
  -newer /usr/bin/systemd -ls 2>/dev/null
# Units newer than systemd binary = recently added (suspicious)
systemctl list-units --type=service --state=running

# Bash profile and rc files (executed on login/shell start):
for f in /etc/profile /etc/profile.d/*.sh /root/.bashrc /root/.bash_profile \
         /root/.profile; do
    echo "=== $f ==="
    cat "$f" 2>/dev/null
done

# PAM modules (rootkit indicator: modified PAM auth):
cat /etc/pam.d/sshd /etc/pam.d/common-auth
ls -la /lib/x86_64-linux-gnu/security/  # Check for unexpected PAM modules

Find recently modified files (most critical):

# Files modified in the last 7 days (adjust -mtime as needed):
find / -mtime -7 -type f -not -path '/proc/*' -not -path '/sys/*' \
  -not -path '/dev/*' -ls 2>/dev/null | sort -k8,9

# Specifically: recently modified SUID binaries (potential rootkit install):
find / -mtime -7 -perm /4000 -type f -ls 2>/dev/null

# Recently created/modified files in web directories (web shells):
find /var/www /srv/www /usr/share/nginx /opt/*/web -mtime -30 \
  -name '*.php' -o -name '*.py' -o -name '*.pl' -o -name '*.jsp' \
  2>/dev/null | xargs ls -la 2>/dev/null

Authentication log analysis:

# Ubuntu/Debian:
grep 'sshd' /var/log/auth.log | tail -200
# Look for: Accepted password/publickey (successful logins), Failed password (brute force)
# Look for: sudo commands executed by attacker

# Find all successful SSH logins:
grep 'Accepted' /var/log/auth.log | \
  awk '{print $1, $2, $3, $9, $11}' | \
  sort -u
# Output: Month Day Time username source_IP

# Find sudo command execution:
grep 'sudo' /var/log/auth.log | grep -v 'pam_unix'
# Shows: which user ran which command as sudo

# RHEL/CentOS:
grep 'sshd\|sudo' /var/log/secure | tail -200

Web server log analysis (web shell detection):

# Apache access log: look for POST requests to PHP files in unusual locations:
grep 'POST' /var/log/apache2/access.log | \
  awk '{print $1, $6, $7, $9}' | \
  grep -E '\.php|\.(cgi|asp|aspx|sh)' | \
  tail -100

# Nginx access log: look for command execution patterns in URLs:
grep -E '(cmd=|exec=|system\(|passthru|shell_exec)' \
  /var/log/nginx/access.log | tail -50

# Find the first time a suspicious PHP file was requested:
awk '{print $7}' /var/log/nginx/access.log | \
  grep -o 'wp-content.*\.php' | sort | uniq -c | sort -rn | head -20
# High-count PHP files in unexpected locations = web shell candidates

# Look for common web shell user agents:
grep -E '(python|curl|wget|Go-http|libwww-perl)' /var/log/nginx/access.log | \
  grep 'POST' | tail -50
# Web shell interactions often use scripted user agents

Detect log tampering:

# Check if auth.log has gaps (attacker may have deleted entries):
ls -lah /var/log/auth.log*  # Check rotation and size
stat /var/log/auth.log       # Mtime = last write time

# auditd can detect log modification if properly configured:
ausearch -k log_modification 2>/dev/null

# Check /var/log/lastlog for login history:
lastlog | head -30  # Shows last login per account

# Check utmp/wtmp for session history:
last -F | head -50  # -F shows full timestamps

Find web shells with content scanning:

# Look for PHP code patterns common in web shells:
grep -r --include='*.php' -l 'eval\|base64_decode\|system\|passthru\|shell_exec\|exec' \
  /var/www/ 2>/dev/null

# More specific: eval(base64_decode(...)) is the classic web shell pattern:
grep -r --include='*.php' 'eval(base64_decode' /var/www/ 2>/dev/null

# Find PHP files with suspicious permissions or in unexpected locations:
find /var/www /tmp /var/tmp -name '*.php' -ls 2>/dev/null
find /var/www -name '*.php' -perm /o+w -ls 2>/dev/null
# World-writable PHP files = extremely suspicious

# Scan with Linux Malware Detect (LMD) or ClamAV:
clamscan -r --infected /var/www/ 2>/dev/null

# Check for hidden files in web directories:
find /var/www -name '.*' -not -name '.htaccess' -ls 2>/dev/null
# Hidden files (starting with .) in web root = suspicious

Rootkit detection:

# Install and run rkhunter:
apt install rkhunter -y
rkhunter --update && rkhunter --check --sk

# Install and run chkrootkit:
apt install chkrootkit -y
chkrootkit

# Check for LD_PRELOAD rootkits (intercept shared library calls):
cat /etc/ld.so.preload  # Should normally be empty
env | grep LD_PRELOAD   # Should normally be empty

# Check for hidden processes using /proc vs ps:
ls /proc/ | grep -E '^[0-9]+$' > /tmp/proc_pids.txt
ps aux | awk '{print $2}' | tail -n +2 | sort -n > /tmp/ps_pids.txt
diff /tmp/proc_pids.txt /tmp/ps_pids.txt
# PIDs in /proc but not in ps = hidden by rootkit