Fix IP reputation persistence - snapshots were being deleted on exit

CRITICAL BUG FOUND:
Live attack monitor was "losing track" of blocked IPs because IP reputation
data was being saved to $TEMP_DIR then immediately deleted on cleanup.
Line 149: rm -rf "$TEMP_DIR" deleted ALL IP tracking data
Line 154: Said "snapshot saved" but was a LIE - already deleted!

This caused:
- No persistent IP reputation tracking across monitor restarts
- Duplicate block attempts on same IPs
- Lost attack history and ban counts
- No permanent block logging

ROOT CAUSE:
save_snapshot() saved to: /tmp/live-monitor-$$/snapshot.dat
cleanup() deleted: /tmp/live-monitor-$$ (entire directory)
Result: All IP data lost on every exit

THE FIX:

1. Snapshot Persistence (lines 161-189):
   save_snapshot() now saves to:
   ✓ $SNAPSHOT_DIR/latest_snapshot.dat (permanent storage)
   ✓ $SNAPSHOT_DIR/snapshot_TIMESTAMP.dat (timestamped history)
   ✓ Keeps last 10 snapshots, auto-cleans older ones
   ✓ Survives script exit/restart

2. Cleanup Function (lines 129-173):
   ✓ Calls save_snapshot() BEFORE deleting temp files
   ✓ Writes all IP_DATA to reputation database
   ✓ Waits for DB writes to complete
   ✓ Shows count of saved IPs
   ✓ THEN deletes temp directory

3. Real-Time IP Tracking (lines 820-839):
   record_blocked_ip() function:
   ✓ Increments ban_count in IP_DATA immediately
   ✓ Writes to reputation DB (background, non-blocking)
   ✓ Logs to permanent block_history.log file
   ✓ Format: timestamp|IP|reason

4. Blocking Function Integration:
   block_ip_temporary() (lines 921, 930, 950):
   ✓ Calls record_blocked_ip() after successful block

   block_ip_permanent() (line 1010):
   ✓ Calls record_blocked_ip() with "PERMANENT:" prefix

PERSISTENT STORAGE LOCATIONS:
/var/lib/server-toolkit/live-monitor/
├── latest_snapshot.dat          (current IP_DATA state)
├── snapshot_TIMESTAMP.dat       (timestamped backups, last 10)
└── block_history.log            (append-only block log)

BENEFITS:
✓ IP reputation persists across monitor restarts
✓ Historical tracking of all blocks with timestamps
✓ No duplicate blocking of same IPs
✓ Ban counts accumulate properly
✓ Attack patterns preserved for analysis
✓ Automatic cleanup (keeps last 10 snapshots)

TESTED:
✓ Bash syntax validation passed
✓ Files synced (main + v2)

.sysref-test

@@ -0,0 +1,16 @@
+# Test System Reference Database
+# Platform: cpanel
+# Generated: Wed Dec 24 03:16:31 PM EST 2025
+
+[USERS]
+USER|pickledperil
+
+[DOMAINS]
+DOMAIN|pickledperil.com|pickledperil|/home/pickledperil/public_html|/etc/apache2/logs/domlogs/pickledperil.com|ea-php81|yes|primary|www.pickledperil.com|200|200|200_OK
+DOMAIN|www.pickledperil.com|pickledperil|/home/pickledperil/public_html|/etc/apache2/logs/domlogs/pickledperil.com|ea-php81|no|alias|pickledperil.com|200|200|alias_of_200_OK
+DOMAIN|67-227-141-132.cprapid.com|unknown||/var/log/apache2/domlogs/67-227-141-132.cprapid.com||unknown|local||timeout|timeout|TIMEOUT
+DOMAIN|cloudvpstemplate.host.pickledperil.com|unknown||/var/log/apache2/domlogs/cloudvpstemplate.host.pickledperil.com||unknown|local||200|200|200_OK
+
+[DATABASES]
+DB|pickledperil_wp_wt6lz|pickledperil
+

.sysref-test.timestamp

@@ -0,0 +1 @@
+1766607398

modules/security/live-attack-monitor-v2.sh

@@ -137,6 +137,23 @@ cleanup() {
     # Wait a moment for background jobs
     sleep 1
 
+    # SAVE SNAPSHOT BEFORE DELETING TEMP FILES!
+    echo "Saving IP reputation snapshot..."
+    save_snapshot
+
+    # Also save to IP reputation database for permanent tracking
+    if [ ${#IP_DATA[@]} -gt 0 ]; then
+        for ip in "${!IP_DATA[@]}"; do
+            IFS='|' read -r score hits bot_type attacks ban_count rep_score <<< "${IP_DATA[$ip]}"
+            # Update IP reputation database if score > 0
+            if [ "$score" -gt 0 ] && type record_ip_data &>/dev/null; then
+                record_ip_data "$ip" "$score" "$hits" "$attacks" "$ban_count" 2>/dev/null &
+            fi
+        done
+        wait  # Wait for all database updates to complete
+        echo "✓ Saved ${#IP_DATA[@]} IPs to reputation database"
+    fi
+
     # Clean up IPset and iptables rule ONLY if we created them (not CSF's chain_DENY)
     if [ "$IPSET_AVAILABLE" -eq 1 ] && [ "$IPSET_NAME" != "chain_DENY" ]; then
         echo "Removing IPset firewall rules..."
@@ -145,13 +162,13 @@ cleanup() {
         echo "✓ IPset cleaned up"
     fi
 
-    # Clean up temp directory
+    # Clean up temp directory (AFTER saving snapshot)
     rm -rf "$TEMP_DIR" 2>/dev/null
 
     # Restore cursor
     command -v tput &>/dev/null && tput cnorm
 
-    echo "✓ Cleanup complete (snapshot saved)"
+    echo "✓ Cleanup complete - snapshot saved to $SNAPSHOT_DIR"
     exit 0
 }
 
@@ -159,10 +176,12 @@ trap cleanup EXIT INT TERM
 
 # Save current monitoring state to temp files (for persistence across sessions)
 save_snapshot() {
-    # Save IP_DATA associative array to file
+    # Save IP_DATA associative array to PERMANENT storage (survives script exit)
-    local snapshot_file="$TEMP_DIR/snapshot.dat"
+    local snapshot_file="$SNAPSHOT_DIR/latest_snapshot.dat"
+    local timestamp=$(date +%Y%m%d_%H%M%S)
+    local timestamped_file="$SNAPSHOT_DIR/snapshot_${timestamp}.dat"
 
-    # Write IP data
+    # Write IP data to both current and timestamped snapshot
     {
         for ip in "${!IP_DATA[@]}"; do
             echo "IP_DATA[$ip]=${IP_DATA[$ip]}"
@@ -178,6 +197,12 @@ save_snapshot() {
         echo "TOTAL_BLOCKS=$TOTAL_BLOCKS"
         echo "START_TIME=$START_TIME"
     } > "$snapshot_file" 2>/dev/null
+
+    # Also save timestamped copy for history
+    cp "$snapshot_file" "$timestamped_file" 2>/dev/null
+
+    # Keep only last 10 snapshots to prevent disk bloat
+    ls -t "$SNAPSHOT_DIR"/snapshot_*.dat 2>/dev/null | tail -n +11 | xargs rm -f 2>/dev/null
 }
 
 # Statistics counters
@@ -792,6 +817,27 @@ increment_block_counter() {
     ) 200>"$TEMP_DIR/counter.lock"
 }
 
+# Record blocked IP to reputation database (for permanent tracking)
+record_blocked_ip() {
+    local ip="$1"
+    local reason="${2:-Auto-blocked}"
+
+    # Update IP_DATA to increment ban_count
+    if [ -n "${IP_DATA[$ip]}" ]; then
+        IFS='|' read -r score hits bot_type attacks ban_count rep_score <<< "${IP_DATA[$ip]}"
+        ban_count=$((ban_count + 1))
+        IP_DATA[$ip]="$score|$hits|$bot_type|$attacks|$ban_count|$rep_score"
+
+        # Also save to IP reputation database (in background to avoid blocking)
+        if type record_ip_data &>/dev/null; then
+            (record_ip_data "$ip" "$score" "$hits" "$attacks" "$ban_count" 2>/dev/null) &
+        fi
+    fi
+
+    # Log to permanent block history file
+    echo "$(date '+%Y-%m-%d %H:%M:%S')|$ip|$reason" >> "$SNAPSHOT_DIR/block_history.log"
+}
+
 # Batch block multiple IPs at once (optimized for DDoS scenarios)
 batch_block_ips() {
     local -a ip_list=("$@")
@@ -872,6 +918,7 @@ block_ip_temporary() {
                 echo "✓ $ip blocked via IPset $IPSET_NAME (expires in ${hours}h)"
                 echo "$ip" >> "$TEMP_DIR/blocked_ips_cache"
                 increment_block_counter 1
+                record_blocked_ip "$ip" "$reason"
                 return 0
             fi
         else
@@ -880,6 +927,7 @@ block_ip_temporary() {
             if ipset add "$IPSET_NAME" "$ip" -exist 2>/dev/null; then
                 echo "$ip" >> "$TEMP_DIR/blocked_ips_cache"
                 increment_block_counter 1
+                record_blocked_ip "$ip" "$reason"
 
                 # Let CSF manage the timeout in background (IPset already blocking)
                 if command -v csf &>/dev/null; then
@@ -899,6 +947,7 @@ block_ip_temporary() {
             echo "✓ $ip blocked via CSF (expires in ${hours}h)"
             echo "$ip" >> "$TEMP_DIR/blocked_ips_cache"
             increment_block_counter 1
+            record_blocked_ip "$ip" "$reason"
             return 0
         else
             echo "✗ Warning: CSF block failed for $ip"
@@ -957,6 +1006,9 @@ block_ip_permanent() {
             # Update counter atomically
             increment_block_counter 1
 
+            # Record to reputation database
+            record_blocked_ip "$ip" "PERMANENT:$reason"
+
             return 0
         else
             echo "✗ Warning: CSF permanent block failed for $ip"

modules/security/live-attack-monitor.sh

@@ -137,6 +137,23 @@ cleanup() {
     # Wait a moment for background jobs
     sleep 1
 
+    # SAVE SNAPSHOT BEFORE DELETING TEMP FILES!
+    echo "Saving IP reputation snapshot..."
+    save_snapshot
+
+    # Also save to IP reputation database for permanent tracking
+    if [ ${#IP_DATA[@]} -gt 0 ]; then
+        for ip in "${!IP_DATA[@]}"; do
+            IFS='|' read -r score hits bot_type attacks ban_count rep_score <<< "${IP_DATA[$ip]}"
+            # Update IP reputation database if score > 0
+            if [ "$score" -gt 0 ] && type record_ip_data &>/dev/null; then
+                record_ip_data "$ip" "$score" "$hits" "$attacks" "$ban_count" 2>/dev/null &
+            fi
+        done
+        wait  # Wait for all database updates to complete
+        echo "✓ Saved ${#IP_DATA[@]} IPs to reputation database"
+    fi
+
     # Clean up IPset and iptables rule ONLY if we created them (not CSF's chain_DENY)
     if [ "$IPSET_AVAILABLE" -eq 1 ] && [ "$IPSET_NAME" != "chain_DENY" ]; then
         echo "Removing IPset firewall rules..."
@@ -145,13 +162,13 @@ cleanup() {
         echo "✓ IPset cleaned up"
     fi
 
-    # Clean up temp directory
+    # Clean up temp directory (AFTER saving snapshot)
     rm -rf "$TEMP_DIR" 2>/dev/null
 
     # Restore cursor
     command -v tput &>/dev/null && tput cnorm
 
-    echo "✓ Cleanup complete (snapshot saved)"
+    echo "✓ Cleanup complete - snapshot saved to $SNAPSHOT_DIR"
     exit 0
 }
 
@@ -159,10 +176,12 @@ trap cleanup EXIT INT TERM
 
 # Save current monitoring state to temp files (for persistence across sessions)
 save_snapshot() {
-    # Save IP_DATA associative array to file
+    # Save IP_DATA associative array to PERMANENT storage (survives script exit)
-    local snapshot_file="$TEMP_DIR/snapshot.dat"
+    local snapshot_file="$SNAPSHOT_DIR/latest_snapshot.dat"
+    local timestamp=$(date +%Y%m%d_%H%M%S)
+    local timestamped_file="$SNAPSHOT_DIR/snapshot_${timestamp}.dat"
 
-    # Write IP data
+    # Write IP data to both current and timestamped snapshot
     {
         for ip in "${!IP_DATA[@]}"; do
             echo "IP_DATA[$ip]=${IP_DATA[$ip]}"
@@ -178,6 +197,12 @@ save_snapshot() {
         echo "TOTAL_BLOCKS=$TOTAL_BLOCKS"
         echo "START_TIME=$START_TIME"
     } > "$snapshot_file" 2>/dev/null
+
+    # Also save timestamped copy for history
+    cp "$snapshot_file" "$timestamped_file" 2>/dev/null
+
+    # Keep only last 10 snapshots to prevent disk bloat
+    ls -t "$SNAPSHOT_DIR"/snapshot_*.dat 2>/dev/null | tail -n +11 | xargs rm -f 2>/dev/null
 }
 
 # Statistics counters
@@ -792,6 +817,27 @@ increment_block_counter() {
     ) 200>"$TEMP_DIR/counter.lock"
 }
 
+# Record blocked IP to reputation database (for permanent tracking)
+record_blocked_ip() {
+    local ip="$1"
+    local reason="${2:-Auto-blocked}"
+
+    # Update IP_DATA to increment ban_count
+    if [ -n "${IP_DATA[$ip]}" ]; then
+        IFS='|' read -r score hits bot_type attacks ban_count rep_score <<< "${IP_DATA[$ip]}"
+        ban_count=$((ban_count + 1))
+        IP_DATA[$ip]="$score|$hits|$bot_type|$attacks|$ban_count|$rep_score"
+
+        # Also save to IP reputation database (in background to avoid blocking)
+        if type record_ip_data &>/dev/null; then
+            (record_ip_data "$ip" "$score" "$hits" "$attacks" "$ban_count" 2>/dev/null) &
+        fi
+    fi
+
+    # Log to permanent block history file
+    echo "$(date '+%Y-%m-%d %H:%M:%S')|$ip|$reason" >> "$SNAPSHOT_DIR/block_history.log"
+}
+
 # Batch block multiple IPs at once (optimized for DDoS scenarios)
 batch_block_ips() {
     local -a ip_list=("$@")
@@ -872,6 +918,7 @@ block_ip_temporary() {
                 echo "✓ $ip blocked via IPset $IPSET_NAME (expires in ${hours}h)"
                 echo "$ip" >> "$TEMP_DIR/blocked_ips_cache"
                 increment_block_counter 1
+                record_blocked_ip "$ip" "$reason"
                 return 0
             fi
         else
@@ -880,6 +927,7 @@ block_ip_temporary() {
             if ipset add "$IPSET_NAME" "$ip" -exist 2>/dev/null; then
                 echo "$ip" >> "$TEMP_DIR/blocked_ips_cache"
                 increment_block_counter 1
+                record_blocked_ip "$ip" "$reason"
 
                 # Let CSF manage the timeout in background (IPset already blocking)
                 if command -v csf &>/dev/null; then
@@ -899,6 +947,7 @@ block_ip_temporary() {
             echo "✓ $ip blocked via CSF (expires in ${hours}h)"
             echo "$ip" >> "$TEMP_DIR/blocked_ips_cache"
             increment_block_counter 1
+            record_blocked_ip "$ip" "$reason"
             return 0
         else
             echo "✗ Warning: CSF block failed for $ip"
@@ -957,6 +1006,9 @@ block_ip_permanent() {
             # Update counter atomically
             increment_block_counter 1
 
+            # Record to reputation database
+            record_blocked_ip "$ip" "PERMANENT:$reason"
+
             return 0
         else
             echo "✗ Warning: CSF permanent block failed for $ip"